{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "7bc8dacdc0d6ebb4",
   "metadata": {},
   "source": [
    "# FenGen个性化联邦学习例子"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "938d2c5d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-08-26T11:38:38.466144Z",
     "start_time": "2024-08-26T11:38:38.457278Z"
    }
   },
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a700aec9",
   "metadata": {},
   "source": [
    "## 在secretflow环境创造3个实体[Alice，Bob，Charlie]，其中 Alice, Bob和Charlie 是三个PYU，Alice和Bob角色是client，Charlie角色是server。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "1c6ab3f20b850118",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/yang/anaconda3/envs/mysf/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n",
      "2024-10-23 10:03:08,660\tINFO util.py:154 -- Missing packages: ['ipywidgets']. Run `pip install -U ipywidgets`, then restart the notebook server for rich notebook output.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The version of SecretFlow: 1.9.0.dev20240726\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/yang/anaconda3/envs/mysf/lib/python3.10/subprocess.py:1796: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock.\n",
      "  self.pid = _posixsubprocess.fork_exec(\n",
      "2024-10-23 10:03:11,837\tINFO worker.py:1724 -- Started a local Ray instance.\n"
     ]
    }
   ],
   "source": [
    "import secretflow as sf\n",
    "\n",
    "# Check the version of your SecretFlow\n",
    "print('The version of SecretFlow: {}'.format(sf.__version__))\n",
    "\n",
    "# In case you have a running secretflow runtime already.\n",
    "sf.shutdown()\n",
    "\n",
    "sf.init(['alice', 'bob', 'charlie'], address='local')\n",
    "alice, bob, charlie = sf.PYU('alice'), sf.PYU('bob'), sf.PYU('charlie')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "461ab9e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "spu = sf.SPU(sf.utils.testing.cluster_def(['alice', 'bob']))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "40435977-c2c6-464c-bdcb-da10063fbed8",
   "metadata": {},
   "source": [
    "## 导入相关依赖"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "403dddd7",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-10-23 10:03:16.561553: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n"
     ]
    }
   ],
   "source": [
    "from secretflow_fl.ml.nn.core.torch import (\n",
    "    metric_wrapper,\n",
    "    optim_wrapper,\n",
    "    BaseModule,\n",
    "    TorchModel,\n",
    ")\n",
    "from secretflow_fl.ml.nn import FLModel\n",
    "from torchmetrics import Accuracy, Precision\n",
    "from secretflow_fl.security.aggregation import SecureAggregator\n",
    "from secretflow_fl.utils.simulation.datasets_fl import load_mnist\n",
    "from torch import nn, optim\n",
    "from torch.nn import functional as F\n",
    "import torch"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f7a71564",
   "metadata": {},
   "source": [
    "## 数据划分，这里模拟数据不平衡2:8分"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "adb0271c",
   "metadata": {},
   "outputs": [],
   "source": [
    "(train_data, train_label), (test_data, test_label) = load_mnist(\n",
    "    parts={alice: 0.2, bob: 0.8},\n",
    "    normalized_x=True,\n",
    "    categorical_y=True,\n",
    "    is_torch=True,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9b00201f",
   "metadata": {},
   "source": [
    "## 定义一个神经网络模型，输出是logit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "f7169dc6",
   "metadata": {},
   "outputs": [],
   "source": [
    "class ConvNet(BaseModule):\n",
    "    \"\"\"Small ConvNet for MNIST.\"\"\"\n",
    "\n",
    "    def __init__(self, kl_div_loss, num_classes):\n",
    "        super(ConvNet, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(1, 3, kernel_size=3)\n",
    "        self.fc_in_dim = 192\n",
    "        self.fc = nn.Linear(self.fc_in_dim, 10)\n",
    "        self.kl_div_loss = kl_div_loss\n",
    "        self.num_classes = num_classes\n",
    "\n",
    "    def forward(self, x, start_layer_idx=0):\n",
    "        if start_layer_idx == -1:\n",
    "            x = self.fc(x)\n",
    "            return x\n",
    "        x = F.relu(F.max_pool2d(self.conv1(x), 3))\n",
    "        x = x.view(-1, self.fc_in_dim)\n",
    "        x = self.fc(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ba14607e",
   "metadata": {},
   "source": [
    "定义神经网络模型的损失函数和优化器"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "18884242",
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_fn = nn.CrossEntropyLoss\n",
    "optim_fn = optim_wrapper(optim.Adam, lr=1e-2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0b21603b",
   "metadata": {},
   "source": [
    "## 准备FedGen相关工作，生成器模型等"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a0da3e36",
   "metadata": {},
   "source": [
    "DiversityLoss是一个自定义的损失函数类"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "e6217205",
   "metadata": {},
   "outputs": [],
   "source": [
    "class DiversityLoss(nn.Module):\n",
    "    \"\"\"\n",
    "    Diversity loss for improving the performance.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, metric):\n",
    "        \"\"\"\n",
    "        Class initializer.\n",
    "        \"\"\"\n",
    "        super().__init__()\n",
    "        self.metric = metric\n",
    "        self.cosine = nn.CosineSimilarity(dim=2)\n",
    "\n",
    "    def compute_distance(self, tensor1, tensor2, metric):\n",
    "        \"\"\"\n",
    "        Compute the distance between two tensors.\n",
    "        \"\"\"\n",
    "        if metric == 'l1':\n",
    "            return torch.abs(tensor1 - tensor2).mean(dim=(2,))\n",
    "        elif metric == 'l2':\n",
    "            return torch.pow(tensor1 - tensor2, 2).mean(dim=(2,))\n",
    "        elif metric == 'cosine':\n",
    "            return 1 - self.cosine(tensor1, tensor2)\n",
    "        else:\n",
    "            raise ValueError(metric)\n",
    "\n",
    "    def pairwise_distance(self, tensor, how):\n",
    "        \"\"\"\n",
    "        Compute the pairwise distances between a Tensor's rows.\n",
    "        \"\"\"\n",
    "        n_data = tensor.size(0)\n",
    "        tensor1 = tensor.expand((n_data, n_data, tensor.size(1)))\n",
    "        tensor2 = tensor.unsqueeze(dim=1)\n",
    "        return self.compute_distance(tensor1, tensor2, how)\n",
    "\n",
    "    def forward(self, noises, layer):\n",
    "        \"\"\"\n",
    "        Forward propagation.\n",
    "        \"\"\"\n",
    "        if len(layer.shape) > 2:\n",
    "            layer = layer.view((layer.size(0), -1))\n",
    "        layer_dist = self.pairwise_distance(layer, how=self.metric)\n",
    "        noise_dist = self.pairwise_distance(noises, how='l2')\n",
    "        return torch.exp(torch.mean(-noise_dist * layer_dist))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "36ed20aa",
   "metadata": {},
   "source": [
    "FedGen需要有generator模型,并一些训练参数相关设置，优化器等"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "abeef2a2",
   "metadata": {},
   "outputs": [],
   "source": [
    "from secretflow_fl.ml.nn.fl.backend.torch.strategy.fed_gen import (\n",
    "    FedGenGeneratorModel,\n",
    "    FedGenActor,\n",
    ")\n",
    "\n",
    "kl_div_loss = nn.KLDivLoss(reduction=\"batchmean\")\n",
    "diversity_loss = DiversityLoss(metric='l1')\n",
    "cross_entropy_loss = nn.CrossEntropyLoss()\n",
    "num_classes = 10\n",
    "generator = FedGenGeneratorModel(\n",
    "    hidden_dimension=256,\n",
    "    latent_dimension=192,\n",
    "    noise_dim=64,\n",
    "    num_classes=num_classes,\n",
    "    loss_fn=loss_fn,\n",
    "    optim_fn=optim_fn,\n",
    "    diversity_loss=diversity_loss,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "436038d0",
   "metadata": {},
   "source": [
    "## 进行联邦学习"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e1d2255b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Create proxy actor <class 'secretflow.device.proxy.ActorFedGenActor'> with party charlie.\n",
      "INFO:root:Create proxy actor <class 'secretflow.device.proxy.Actor_Masker'> with party alice.\n",
      "INFO:root:Create proxy actor <class 'secretflow.device.proxy.Actor_Masker'> with party bob.\n",
      "INFO:root:Create proxy actor <class 'abc.ActorPYUFedGen'> with party alice.\n",
      "INFO:root:Create proxy actor <class 'abc.ActorPYUFedGen'> with party bob.\n",
      "INFO:root:FL Train Params: {'x': FedNdarray(partitions={PYURuntime(alice): <secretflow.device.device.pyu.PYUObject object at 0x7fba63bc4130>, PYURuntime(bob): <secretflow.device.device.pyu.PYUObject object at 0x7fba63bc4b50>}, partition_way=<PartitionWay.HORIZONTAL: 'horizontal'>), 'y': FedNdarray(partitions={PYURuntime(alice): <secretflow.device.device.pyu.PYUObject object at 0x7fba63bc54b0>, PYURuntime(bob): <secretflow.device.device.pyu.PYUObject object at 0x7fba63bc4070>}, partition_way=<PartitionWay.HORIZONTAL: 'horizontal'>), 'batch_size': 32, 'batch_sampling_rate': None, 'epochs': 20, 'verbose': 1, 'callbacks': None, 'validation_data': (FedNdarray(partitions={PYURuntime(alice): <secretflow.device.device.pyu.PYUObject object at 0x7fba63bc5420>, PYURuntime(bob): <secretflow.device.device.pyu.PYUObject object at 0x7fba63bc5630>}, partition_way=<PartitionWay.HORIZONTAL: 'horizontal'>), FedNdarray(partitions={PYURuntime(alice): <secretflow.device.device.pyu.PYUObject object at 0x7fba63bc5780>, PYURuntime(bob): <secretflow.device.device.pyu.PYUObject object at 0x7fba63bc59f0>}, partition_way=<PartitionWay.HORIZONTAL: 'horizontal'>)), 'shuffle': False, 'class_weight': None, 'sample_weight': None, 'validation_freq': 1, 'aggregate_freq': 1, 'label_decoder': None, 'max_batch_size': 20000, 'prefetch_buffer_size': None, 'sampler_method': 'batch', 'random_seed': 50467, 'dp_spent_step_freq': None, 'audit_log_dir': None, 'dataset_builder': None, 'wait_steps': 100, 'self': <secretflow.ml.nn.fl.fl_model.FLModel object at 0x7fba63a52770>}\n",
      "\u001b[36m(ActorFedGenActor pid=596183)\u001b[0m 2024-10-23 10:03:34.051121: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
      "Train Processing: :   0%|                                                                                                                                   | 0/375 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[36m(ActorFedGenActor pid=596183)\u001b[0m /home/yang/anaconda3/envs/mysf/lib/python3.10/site-packages/torch/nn/functional.py:2943: UserWarning: reduction: 'mean' divides the total loss by both the batch size and the support size.'batchmean' divides only by the batch size, and aligns with the KL div math definition.'mean' will be changed to behave the same as 'batchmean' in the next major release.\n",
      "\u001b[36m(ActorFedGenActor pid=596183)\u001b[0m   warnings.warn(\n",
      "\u001b[36m(pid=596362)\u001b[0m 2024-10-23 10:03:34.443714: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\u001b[32m [repeated 2x across cluster] (Ray deduplicates logs by default. Set RAY_DEDUP_LOGS=0 to disable log deduplication, or see https://docs.ray.io/en/master/ray-observability/ray-logging.html#log-deduplication for more options.)\u001b[0m\n",
      "Train Processing: : 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋| 374/375 [01:13<00:00,  4.48it/s]/home/yang/PycharmProjects/secretflow/secretflow/ml/nn/metrics.py:62: UserWarning: Please pay attention to local metrics, global only do naive aggregation.\n",
      "  warnings.warn(\n",
      "2024-10-23 10:04:54.650508: W tensorflow/core/common_runtime/gpu/gpu_device.cc:1956] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.\n",
      "Skipping registering GPU devices...\n",
      "Train Processing: : 100%|▉| 374/375 [01:14<00:00,  5.02it/s, {'multiclassaccuracy': 0.7865833, 'multiclassprecision': 0.7865833, 'val_multiclassaccuracy': 0.8935, 'val_multiclasspr\n",
      "Train Processing: :   0%|                                                                                                                                   | 0/375 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 2/20\n",
      "\u001b[36m(ActorPYUFedGen pid=596361)\u001b[0m {'train-loss': 7.240610122680664, 'train_multiclassaccuracy': tensor(0.7905), 'train_multiclassprecision': tensor(0.7905), 'val_eval_multiclassaccuracy': tensor(0.8730), 'val_eval_multiclassprecision': tensor(0.8730)}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Train Processing: : 100%|▉| 374/375 [00:38<00:00,  9.78it/s, {'multiclassaccuracy': 0.91835415, 'multiclassprecision': 0.91835415, 'val_multiclassaccuracy': 0.9181875, 'val_multicl\n",
      "Train Processing: :   0%|▎                                                                                                                          | 1/375 [00:00<00:56,  6.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 3/20\n",
      "\u001b[36m(ActorPYUFedGen pid=596361)\u001b[0m {'train-loss': 3.871393918991089, 'train_multiclassaccuracy': tensor(0.9191), 'train_multiclassprecision': tensor(0.9191), 'val_eval_multiclassaccuracy': tensor(0.9020), 'val_eval_multiclassprecision': tensor(0.9020)}\u001b[32m [repeated 2x across cluster]\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Train Processing: : 100%|▉| 374/375 [00:37<00:00,  9.90it/s, {'multiclassaccuracy': 0.93016666, 'multiclassprecision': 0.93016666, 'val_multiclassaccuracy': 0.9263125, 'val_multicl\n",
      "Train Processing: :   0%|▎                                                                                                                          | 1/375 [00:00<00:58,  6.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 4/20\n",
      "\u001b[36m(ActorPYUFedGen pid=596361)\u001b[0m {'train-loss': 3.469554901123047, 'train_multiclassaccuracy': tensor(0.9317), 'train_multiclassprecision': tensor(0.9317), 'val_eval_multiclassaccuracy': tensor(0.9125), 'val_eval_multiclassprecision': tensor(0.9125)}\u001b[32m [repeated 2x across cluster]\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Train Processing: : 100%|▉| 374/375 [00:34<00:00, 10.73it/s, {'multiclassaccuracy': 0.93403125, 'multiclassprecision': 0.93403125, 'val_multiclassaccuracy': 0.928125, 'val_multicla\n",
      "Train Processing: :   0%|▎                                                                                                                          | 1/375 [00:00<01:00,  6.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 5/20\n",
      "\u001b[36m(ActorPYUFedGen pid=596361)\u001b[0m {'train-loss': 3.194899559020996, 'train_multiclassaccuracy': tensor(0.9355), 'train_multiclassprecision': tensor(0.9355), 'val_eval_multiclassaccuracy': tensor(0.9140), 'val_eval_multiclassprecision': tensor(0.9140)}\u001b[32m [repeated 2x across cluster]\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Train Processing: : 100%|▉| 374/375 [00:34<00:00, 11.00it/s, {'multiclassaccuracy': 0.9375833, 'multiclassprecision': 0.9375833, 'val_multiclassaccuracy': 0.9320625, 'val_multiclas\n",
      "Train Processing: :   0%|▎                                                                                                                          | 1/375 [00:00<00:59,  6.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 6/20\n",
      "\u001b[36m(ActorPYUFedGen pid=596361)\u001b[0m {'train-loss': 3.116774559020996, 'train_multiclassaccuracy': tensor(0.9392), 'train_multiclassprecision': tensor(0.9392), 'val_eval_multiclassaccuracy': tensor(0.9205), 'val_eval_multiclassprecision': tensor(0.9205)}\u001b[32m [repeated 2x across cluster]\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Train Processing: : 100%|▉| 374/375 [00:32<00:00, 11.39it/s, {'multiclassaccuracy': 0.93965626, 'multiclassprecision': 0.93965626, 'val_multiclassaccuracy': 0.9373125, 'val_multicl\n",
      "Train Processing: :   0%|▎                                                                                                                          | 1/375 [00:00<00:58,  6.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 7/20\n",
      "\u001b[36m(ActorPYUFedGen pid=596361)\u001b[0m {'train-loss': 2.8242197036743164, 'train_multiclassaccuracy': tensor(0.9411), 'train_multiclassprecision': tensor(0.9411), 'val_eval_multiclassaccuracy': tensor(0.9265), 'val_eval_multiclassprecision': tensor(0.9265)}\u001b[32m [repeated 2x across cluster]\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Train Processing: : 100%|▉| 374/375 [00:33<00:00, 11.06it/s, {'multiclassaccuracy': 0.9405625, 'multiclassprecision': 0.9405625, 'val_multiclassaccuracy': 0.938625, 'val_multiclass\n",
      "Train Processing: :   0%|▎                                                                                                                          | 1/375 [00:00<01:12,  5.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 8/20\n",
      "\u001b[36m(ActorPYUFedGen pid=596361)\u001b[0m {'train-loss': 2.6636083126068115, 'train_multiclassaccuracy': tensor(0.9416), 'train_multiclassprecision': tensor(0.9416), 'val_eval_multiclassaccuracy': tensor(0.9280), 'val_eval_multiclassprecision': tensor(0.9280)}\u001b[32m [repeated 2x across cluster]\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Train Processing: : 100%|▉| 374/375 [00:34<00:00, 10.95it/s, {'multiclassaccuracy': 0.9417083, 'multiclassprecision': 0.9417083, 'val_multiclassaccuracy': 0.9391875, 'val_multiclas\n",
      "Train Processing: :   0%|                                                                                                                                   | 0/375 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 9/20\n",
      "\u001b[36m(ActorPYUFedGen pid=596361)\u001b[0m {'train-loss': 2.566133499145508, 'train_multiclassaccuracy': tensor(0.9428), 'train_multiclassprecision': tensor(0.9428), 'val_eval_multiclassaccuracy': tensor(0.9280), 'val_eval_multiclassprecision': tensor(0.9280)}\u001b[32m [repeated 2x across cluster]\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Train Processing: : 100%|▉| 374/375 [00:34<00:00, 10.75it/s, {'multiclassaccuracy': 0.9431771, 'multiclassprecision': 0.9431771, 'val_multiclassaccuracy': 0.9399375, 'val_multiclas\n",
      "Train Processing: :   0%|▎                                                                                                                          | 1/375 [00:00<01:04,  5.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 10/20\n",
      "\u001b[36m(ActorPYUFedGen pid=596361)\u001b[0m {'train-loss': 2.377262592315674, 'train_multiclassaccuracy': tensor(0.9445), 'train_multiclassprecision': tensor(0.9445), 'val_eval_multiclassaccuracy': tensor(0.9290), 'val_eval_multiclassprecision': tensor(0.9290)}\u001b[32m [repeated 2x across cluster]\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Train Processing: : 100%|▉| 374/375 [00:37<00:00,  9.99it/s, {'multiclassaccuracy': 0.9437916, 'multiclassprecision': 0.9437916, 'val_multiclassaccuracy': 0.940125, 'val_multiclass\n",
      "Train Processing: :   0%|▎                                                                                                                          | 1/375 [00:00<01:00,  6.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 11/20\n",
      "\u001b[36m(ActorPYUFedGen pid=596361)\u001b[0m {'train-loss': 2.2059271335601807, 'train_multiclassaccuracy': tensor(0.9450), 'train_multiclassprecision': tensor(0.9450), 'val_eval_multiclassaccuracy': tensor(0.9280), 'val_eval_multiclassprecision': tensor(0.9280)}\u001b[32m [repeated 2x across cluster]\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Train Processing: : 100%|▉| 374/375 [00:35<00:00, 10.61it/s, {'multiclassaccuracy': 0.94498956, 'multiclassprecision': 0.94498956, 'val_multiclassaccuracy': 0.94025, 'val_multiclas\n",
      "Train Processing: :   0%|▎                                                                                                                          | 1/375 [00:00<01:04,  5.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 12/20\n",
      "\u001b[36m(ActorPYUFedGen pid=596361)\u001b[0m {'train-loss': 2.115673065185547, 'train_multiclassaccuracy': tensor(0.9463), 'train_multiclassprecision': tensor(0.9463), 'val_eval_multiclassaccuracy': tensor(0.9280), 'val_eval_multiclassprecision': tensor(0.9280)}\u001b[32m [repeated 2x across cluster]\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Train Processing: : 100%|▉| 374/375 [00:34<00:00, 10.94it/s, {'multiclassaccuracy': 0.9458333, 'multiclassprecision': 0.9458333, 'val_multiclassaccuracy': 0.9413125, 'val_multiclas\n",
      "Train Processing: :   0%|▎                                                                                                                          | 1/375 [00:00<00:58,  6.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 13/20\n",
      "\u001b[36m(ActorPYUFedGen pid=596361)\u001b[0m {'train-loss': 1.943589448928833, 'train_multiclassaccuracy': tensor(0.9469), 'train_multiclassprecision': tensor(0.9469), 'val_eval_multiclassaccuracy': tensor(0.9300), 'val_eval_multiclassprecision': tensor(0.9300)}\u001b[32m [repeated 2x across cluster]\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Train Processing: : 100%|▉| 374/375 [00:35<00:00, 10.45it/s, {'multiclassaccuracy': 0.9465208, 'multiclassprecision': 0.9465208, 'val_multiclassaccuracy': 0.94306254, 'val_multicla\n",
      "Train Processing: :   0%|▎                                                                                                                          | 1/375 [00:00<01:04,  5.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 14/20\n",
      "\u001b[36m(ActorPYUFedGen pid=596361)\u001b[0m {'train-loss': 2.120236396789551, 'train_multiclassaccuracy': tensor(0.9481), 'train_multiclassprecision': tensor(0.9481), 'val_eval_multiclassaccuracy': tensor(0.9330), 'val_eval_multiclassprecision': tensor(0.9330)}\u001b[32m [repeated 2x across cluster]\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Train Processing: : 100%|▉| 374/375 [00:33<00:00, 11.03it/s, {'multiclassaccuracy': 0.9479271, 'multiclassprecision': 0.9479271, 'val_multiclassaccuracy': 0.9443125, 'val_multiclas\n",
      "Train Processing: :   0%|▎                                                                                                                          | 1/375 [00:00<00:57,  6.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 15/20\n",
      "\u001b[36m(ActorPYUFedGen pid=596361)\u001b[0m {'train-loss': 1.9986705780029297, 'train_multiclassaccuracy': tensor(0.9499), 'train_multiclassprecision': tensor(0.9499), 'val_eval_multiclassaccuracy': tensor(0.9340), 'val_eval_multiclassprecision': tensor(0.9340)}\u001b[32m [repeated 2x across cluster]\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Train Processing: : 100%|▉| 374/375 [00:32<00:00, 11.39it/s, {'multiclassaccuracy': 0.94815624, 'multiclassprecision': 0.94815624, 'val_multiclassaccuracy': 0.94449997, 'val_multic\n",
      "Train Processing: :   0%|▎                                                                                                                          | 1/375 [00:00<01:00,  6.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 16/20\n",
      "\u001b[36m(ActorPYUFedGen pid=596361)\u001b[0m {'train-loss': 1.9143987894058228, 'train_multiclassaccuracy': tensor(0.9501), 'train_multiclassprecision': tensor(0.9501), 'val_eval_multiclassaccuracy': tensor(0.9335), 'val_eval_multiclassprecision': tensor(0.9335)}\u001b[32m [repeated 2x across cluster]\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Train Processing: : 100%|▉| 374/375 [00:33<00:00, 11.31it/s, {'multiclassaccuracy': 0.94853127, 'multiclassprecision': 0.94853127, 'val_multiclassaccuracy': 0.945125, 'val_multicla\n",
      "Train Processing: :   0%|▎                                                                                                                          | 1/375 [00:00<00:56,  6.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 17/20\n",
      "\u001b[36m(ActorPYUFedGen pid=596361)\u001b[0m {'train-loss': 1.7454423904418945, 'train_multiclassaccuracy': tensor(0.9503), 'train_multiclassprecision': tensor(0.9503), 'val_eval_multiclassaccuracy': tensor(0.9340), 'val_eval_multiclassprecision': tensor(0.9340)}\u001b[32m [repeated 2x across cluster]\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Train Processing: : 100%|▉| 374/375 [00:32<00:00, 11.34it/s, {'multiclassaccuracy': 0.94883335, 'multiclassprecision': 0.94883335, 'val_multiclassaccuracy': 0.94568753, 'val_multic\n",
      "Train Processing: :   0%|▎                                                                                                                          | 1/375 [00:00<00:59,  6.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 18/20\n",
      "\u001b[36m(ActorPYUFedGen pid=596361)\u001b[0m {'train-loss': 1.7196615934371948, 'train_multiclassaccuracy': tensor(0.9505), 'train_multiclassprecision': tensor(0.9505), 'val_eval_multiclassaccuracy': tensor(0.9350), 'val_eval_multiclassprecision': tensor(0.9350)}\u001b[32m [repeated 2x across cluster]\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Train Processing: : 100%|▉| 374/375 [00:33<00:00, 11.32it/s, {'multiclassaccuracy': 0.94934374, 'multiclassprecision': 0.94934374, 'val_multiclassaccuracy': 0.94568753, 'val_multic\n",
      "Train Processing: :   0%|▎                                                                                                                          | 1/375 [00:00<00:58,  6.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 19/20\n",
      "\u001b[36m(ActorPYUFedGen pid=596361)\u001b[0m {'train-loss': 1.5121980905532837, 'train_multiclassaccuracy': tensor(0.9510), 'train_multiclassprecision': tensor(0.9510), 'val_eval_multiclassaccuracy': tensor(0.9340), 'val_eval_multiclassprecision': tensor(0.9340)}\u001b[32m [repeated 2x across cluster]\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Train Processing: : 100%|▉| 374/375 [00:33<00:00, 11.19it/s, {'multiclassaccuracy': 0.94985414, 'multiclassprecision': 0.94985414, 'val_multiclassaccuracy': 0.9450625, 'val_multicl\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[36m(ActorPYUFedGen pid=596361)\u001b[0m {'train-loss': 1.4355396032333374, 'train_multiclassaccuracy': tensor(0.9516), 'train_multiclassprecision': tensor(0.9516), 'val_eval_multiclassaccuracy': tensor(0.9330), 'val_eval_multiclassprecision': tensor(0.9330)}\u001b[32m [repeated 2x across cluster]\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Train Processing: :   0%|▎                                                                                                                          | 1/375 [00:00<00:58,  6.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 20/20\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Train Processing: : 100%|▉| 374/375 [00:34<00:00, 10.99it/s, {'multiclassaccuracy': 0.94983333, 'multiclassprecision': 0.94983333, 'val_multiclassaccuracy': 0.9460625, 'val_multicl\n"
     ]
    }
   ],
   "source": [
    "from secretflow_fl.security.aggregation.stateful_fedgen_aggregator import (\n",
    "    StatefulFedGenAggregator,\n",
    ")\n",
    "\n",
    "net = ConvNet(diversity_loss, 20)\n",
    "\n",
    "model_def = TorchModel(\n",
    "    model_fn=ConvNet,\n",
    "    loss_fn=loss_fn,\n",
    "    optim_fn=optim_fn,\n",
    "    metrics=[\n",
    "        metric_wrapper(Accuracy, task=\"multiclass\", num_classes=10, average='micro'),\n",
    "        metric_wrapper(Precision, task=\"multiclass\", num_classes=10, average='micro'),\n",
    "    ],\n",
    "    kl_div_loss=kl_div_loss,\n",
    "    num_classes=num_classes,\n",
    ")\n",
    "\n",
    "server_actor = FedGenActor(device=charlie, generator=generator)\n",
    "device_list = [alice, bob]\n",
    "aggregator = StatefulFedGenAggregator(charlie, [alice, bob], server_actor)\n",
    "# spcify params\n",
    "fl_model = FLModel(\n",
    "    server=charlie,\n",
    "    device_list=device_list,\n",
    "    model=model_def,\n",
    "    strategy=\"fed_gen\",  # fl strategy\n",
    "    backend=\"torch\",  # backend support ['tensorflow', 'torch']\n",
    "    aggregator=aggregator,\n",
    "    generator=generator,\n",
    ")\n",
    "history = fl_model.fit(\n",
    "    train_data,\n",
    "    train_label,\n",
    "    validation_data=(test_data, test_label),\n",
    "    epochs=20,\n",
    "    batch_size=32,\n",
    "    aggregate_freq=1,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "285ef2c4",
   "metadata": {},
   "source": [
    "## 绘制结果"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "f31086b7-2159-4e21-b8a3-75a3a79efae8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[36m(ActorPYUFedGen pid=596361)\u001b[0m {'train-loss': 1.4233253002166748, 'train_multiclassaccuracy': tensor(0.9514), 'train_multiclassprecision': tensor(0.9514), 'val_eval_multiclassaccuracy': tensor(0.9340), 'val_eval_multiclassprecision': tensor(0.9340)}\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkAAAAHHCAYAAABXx+fLAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABlR0lEQVR4nO3deXhTVeI+8DdJm6T7QvdSWzbZKchSWQYZqVTACrghIJSKMPgDBTvOCFioyBc6bkwVEVwAHQVBFNQRBweLiMg6LajIIgiyFLoCTZuSNE3O7480KaFpaUpW+n6e5z5JTs69ObeXmNdzzz1XIoQQICIiImpBpK5uABEREZGzMQARERFRi8MARERERC0OAxARERG1OAxARERE1OIwABEREVGLwwBERERELQ4DEBEREbU4DEBERETU4jAAEZHb2bFjByQSCXbs2GHzuu+//z4kEgn++OMPu7eLiG4dDEBEtxhTALC2zJkzx1wvISEB9913X6Pbmjx5MiQSCQIDA3H16tV67584ccK87VdffdXu+0JE5Cherm4AETnGiy++iDZt2liUdevWzebteHl5oaqqCv/+97/xyCOPWLy3du1aKJVKaDSam2orEZGzMQAR3aKGDx+OPn363PR2FAoFBg4ciI8//rheAFq3bh1GjhyJzz777KY/hxpXVVUFX19fVzeD6JbBU2BEdEPjx4/Hf/7zH1y5csVcduDAAZw4cQLjx4+3us6pU6fw8MMPIzQ0FL6+vrjzzjuxZcuWevXOnz+P0aNHw8/PDxEREXjmmWeg1WqtbnPfvn249957ERQUBF9fX9x111348ccfm7VPP//8MyZPnoy2bdtCqVQiKioKjz/+OMrKyurVLSgowJQpUxATEwOFQoE2bdrgySefRHV1tbnOlStX8MwzzyAhIQEKhQKtW7fGpEmTUFpaCqDhsUnWxjsNGTIE3bp1Q15eHgYPHgxfX1/MmzcPAPDFF19g5MiR5ra0a9cOixYtgl6vt/r3GjFiBEJCQuDn54cePXrg9ddfBwCsWbMGEokEBw8erLfekiVLIJPJUFBQYPPflchTsAeI6BZVXl5u/vE1CQsLa9a2HnjgAUyfPh2bNm3C448/DsDY+9OpUyfccccd9eoXFRVhwIABqKqqwtNPP41WrVrhgw8+wP33349PP/0UY8aMAQBcvXoVQ4cOxdmzZ/H0008jJiYGH374IbZv315vm9u3b8fw4cPRu3dvZGVlQSqVYs2aNbj77rvxww8/oF+/fjbt07Zt23Dq1Cmkp6cjKioKv/76K9555x38+uuv2Lt3LyQSCQDgwoUL6NevH65cuYJp06ahU6dOKCgowKeffoqqqirI5XJUVlbiT3/6E44ePYrHH38cd9xxB0pLS/Hll1/i/Pnzzfq7l5WVYfjw4Xj00Ufx2GOPITIyEoAxSPn7+yMjIwP+/v7Yvn07FixYAJVKhVdeecVi/+677z5ER0dj1qxZiIqKwtGjR/HVV19h1qxZeOihhzBjxgysXbsWvXr1svjstWvXYsiQIYiNjbW53UQeQxDRLWXNmjUCgNXlWvHx8WLkyJGNbistLU34+fkJIYR46KGHxNChQ4UQQuj1ehEVFSUWLlwoTp8+LQCIV155xbze7NmzBQDxww8/mMsqKipEmzZtREJCgtDr9UIIIXJycgQA8cknn5jrqdVq0b59ewFAfPfdd0IIIQwGg+jQoYNISUkRBoPBXLeqqkq0adNG3HPPPfX2//Tp043uW1VVVb2yjz/+WAAQO3fuNJdNmjRJSKVSceDAgXr1TW1ZsGCBACA2bdrUYJ2G2vXdd99Z7KsQQtx1110CgFi5cmWT2v2Xv/xF+Pr6Co1GI4QQoqamRrRp00bEx8eLy5cvW22PEEKMGzdOxMTEmI+HEELk5+cLAGLNmjX1PofoVsJTYES3qOXLl2Pbtm0Wy80YP348duzYgcLCQmzfvh2FhYUNnv76+uuv0a9fPwwaNMhc5u/vj2nTpuGPP/7AkSNHzPWio6Px0EMPmev5+vpi2rRpFts7dOiQ+XRbWVkZSktLUVpaCrVajaFDh2Lnzp0wGAw27Y+Pj4/5uUajQWlpKe68804AQH5+PgDAYDDg888/R2pqqtXxVKZeos8++wyJiYnmni1rdWylUCiQnp7eaLsrKipQWlqKP/3pT6iqqsKxY8cAAAcPHsTp06cxe/ZsBAcHN9ieSZMm4cKFC/juu+/MZWvXroWPjw8efPDBZrWbyFPwFBjRLapfv352GQRtMmLECAQEBGDDhg04dOgQ+vbti/bt21udb+fMmTNISkqqV965c2fz+926dcOZM2fQvn37eiGhY8eOFq9PnDgBAEhLS2uwfeXl5QgJCWny/ly6dAkLFy7E+vXrUVxcXG9bAFBSUgKVSnXDq+d+//13uweG2NhYyOXyeuW//vorMjMzsX37dqhUKov3TO3+/fffAdz4qr977rkH0dHRWLt2LYYOHQqDwYCPP/4Yo0aNQkBAgJ32hMg9MQARUZMoFAo88MAD+OCDD3Dq1Cm88MILTvtsU+/OK6+8gp49e1qt4+/vb9M2H3nkEezevRt/+9vf0LNnT/j7+8NgMODee++1uTepKRrqCbI2eBmw7OkxuXLlCu666y4EBgbixRdfRLt27aBUKpGfn4/nnnvO5nbLZDKMHz8e7777Lt566y38+OOPuHDhAh577DGbtkPkiRiAiKjJxo8fj9WrV0MqleLRRx9tsF58fDyOHz9er9x0iiY+Pt78ePjwYQghLALC9eu2a9cOABAYGIjk5OSb3o/Lly8jNzcXCxcuxIIFC8zlpp4mk/DwcAQGBuLw4cONbq9du3Y3rGPqnbr2SjrA2BvWVDt27EBZWRk2bdqEwYMHm8tPnz5drz0AcPjw4Rv+vSZNmoTXXnsN//73v/Gf//wH4eHhSElJaXKbiDwVxwARUZP9+c9/xqJFi/Dmm28iKiqqwXojRozA/v37sWfPHnOZWq3GO++8g4SEBHTp0sVc78KFC/j000/N9aqqqvDOO+9YbK93795o164dXn31VVRWVtb7vJKSEpv2QyaTAQCEEBblOTk5Fq+lUilGjx6Nf//73/jf//5Xbzum9R988EH89NNP2Lx5c4N1TKFk586d5vf0en29fbW13dXV1Xjrrbcs6t1xxx1o06YNcnJy6gWu6/e5R48e6NGjB9577z189tlnePTRR+Hlxf83plsf/5UTtWAnT57E//3f/9Ur79WrF0aOHFmvXCqVIjMz84bbnTNnDj7++GMMHz4cTz/9NEJDQ/HBBx/g9OnT+OyzzyCVGv/fa+rUqXjzzTcxadIk5OXlITo6Gh9++GG9Cf+kUinee+89DB8+HF27dkV6ejpiY2NRUFCA7777DoGBgfj3v//d5P0ODAzE4MGD8fLLL0On0yE2Nhb//e9/6/WkAMY5cf773//irrvuwrRp09C5c2dcvHgRGzduxK5duxAcHIy//e1v+PTTT/Hwww/j8ccfR+/evXHp0iV8+eWXWLlyJRITE9G1a1fceeedmDt3Li5duoTQ0FCsX78eNTU1TW73gAEDEBISgrS0NDz99NOQSCT48MMP64UaqVSKFStWIDU1FT179kR6ejqio6Nx7Ngx/Prrr/jmm28s6k+aNAnPPvssAPD0F7UcLrwCjYgcwHS5tbXLtq8VHx/f4OXyU6ZMEUJYXgbfEGuXwQshxO+//y4eeughERwcLJRKpejXr5/46quv6q1/5swZcf/99wtfX18RFhYmZs2aJbZu3Vrv0nAhhDh48KB44IEHRKtWrYRCoRDx8fHikUceEbm5ufX2/0aXwZ8/f16MGTNGBAcHi6CgIPHwww+LCxcuCAAiKyurXhsnTZokwsPDhUKhEG3bthUzZswQWq3WXKesrEzMnDlTxMbGCrlcLlq3bi3S0tJEaWmpxd8kOTlZKBQKERkZKebNmye2bdtm9TL4rl27Wm33jz/+KO68807h4+MjYmJixN///nfxzTffWP177dq1S9xzzz0iICBA+Pn5iR49eohly5bV2+bFixeFTCYTt99+e6N/M6JbiUSI6/7XgYiIWpTS0lJER0djwYIFmD9/vqubQ+QUHANERNTCvf/++9Dr9Zg4caKrm0LkNBwDRETUQm3fvh1HjhzB4sWLMXr0aCQkJLi6SUROw1NgREQt1JAhQ7B7924MHDgQH330Ee/9RS0KAxARERG1OBwDRERERC0OAxARERG1OBwEbYXBYMCFCxcQEBDQ7Ds5ExERkXMJIVBRUYGYmBjzhKsNYQCy4sKFC4iLi3N1M4iIiKgZzp07h9atWzdahwHIioCAAADGP2BgYKCLW0NERERNoVKpEBcXZ/4dbwwDkBWm016BgYEMQERERB6mKcNXOAiaiIiIWhwGICIiImpxGICIiIioxeEYoJug1+uh0+lc3QyP5e3tDZlM5upmEBFRC8QA1AxCCBQWFuLKlSuuborHCw4ORlRUFOdbIiIip2IAagZT+ImIiICvry9/vJtBCIGqqioUFxcDAKKjo13cIiIiakkYgGyk1+vN4adVq1aubo5H8/HxAQAUFxcjIiKCp8OIiMhpOAjaRqYxP76+vi5uya3B9HfkWCoiInImBqBm4mkv++DfkYiIXIEBiIiIiFocBiC6KQkJCcjJyXF1M4iIiGzCANRCSCSSRpcXXnihWds9cOAApk2bZt/GEhERORivAmshLl68aH6+YcMGLFiwAMePHzeX+fv7m58LIaDX6+HldeN/HuHh4fZtKBERuYxOb0BVtR5V1TXQGwSAurGa147YNA3flNSW1r2GxZOG3pdIJJB7SeGvcF0MYQBqIaKioszPg4KCIJFIzGU7duzAn//8Z3z99dfIzMzEL7/8gv/+97+Ii4tDRkYG9u7dC7Vajc6dOyM7OxvJycnmbSUkJGD27NmYPXs2AOM/6nfffRdbtmzBN998g9jYWLz22mu4//77nbq/RES3KoNBQFOjNwYVrR5Vupq659W1z6stn1+troG6Wo+r1Xqoa8tNz69eU1+nF07bj/sTY/DGuF5O+7zrMQDZgRACV3V6p3+uj7fMrldRzZkzB6+++iratm2LkJAQnDt3DiNGjMDixYuhUCjwr3/9C6mpqTh+/Dhuu+22BrezcOFCvPzyy3jllVewbNkyTJgwAWfOnEFoaKjd2kpE5M6EENDWGIzhQmcMIFXVeqi1ely9PrDoTM/r3qtXT2cMKsZyx//eyKQSeEmNvy/mSCRMD6J2H6/Z32v22/K1Q5t5UxiA7OCqTo8uC75x+uceeTEFvnL7HcIXX3wR99xzj/l1aGgoEhMTza8XLVqEzZs348svv8TMmTMb3M7kyZMxbtw4AMCSJUvwxhtvYP/+/bj33nvt1lYiousZDMbQoa3RQ1tjgEZnfNTqDNDU6KHVGd/T6OrqaHV66PQC1XoDtDUG6PQGVNcYF9Nzrd7ytem5tsaAaqvvGbfnDL5yWe3iBV+5DD5yGfzkXvAxlzf8XmP15DKpw6YpEW6SihiAyKxPnz4WrysrK/HCCy9gy5YtuHjxImpqanD16lWcPXu20e306NHD/NzPzw+BgYHmW14Q0a1BCIHLVTpcUmtRXSNQYzD+8NfoDagxCOj0BtTo68pNr3WG2sfaejX62vVqy6uvW6/6hoHGGGJMYcQdyWVS+Cpk8PWuDSEKL/h414YOhRd8r3tuquMrl8HHu7a+ldCi9JJBKvW8udTcZf43BiA78PGW4ciLKS75XHvy8/OzeP3ss89i27ZtePXVV9G+fXv4+PjgoYceQnV1daPb8fb2tngtkUhgMLjnf5iIyDqDQaCkUovzl6+i4MpVnL9chYLa56bHqmrnn/pvKqkEUHrLoPSWQeElrV1kUHobHxXexjK5lxRyWe2jlxTetc8VMsvXFvWs1JfLjNuzqO8lha+3DF4yXnDtjhiA7EAikdj1VJS7+PHHHzF58mSMGTMGgLFH6I8//nBto4jILnR6AwrLNeaAYww1VebXF69omtSjEuTjDW+ZFN4yCbxkEuNzqRReMgm8ZFJ4S+vKvaTGMrms9n1p3Xp1z43reMukxrrm8CKtCzPeMihrHxt6j6GDbuTW+9Umu+nQoQM2bdqE1NRUSCQSzJ8/nz05RG5Oo9OjQlODCo0OFZoaXK6qxoUrGhRcsezBKVRpYLjBUAypBIgO8kFssA9iQ3zQOqTueWywD2KCfaC0c080kbMwAFGDli5discffxwDBgxAWFgYnnvuOahUKlc3i+iWVaM31IaXGqg0OqhqQ4zqqs5cXnFNed3rusfqmqb/T4pcJkVMsBKtQ3wtgo0p7EQFKtmTQvYhBKCtANQlgLrU+OgfCcT1dVmTJMJdhmO7EZVKhaCgIJSXlyMwMNDiPY1Gg9OnT6NNmzZQKpUuauGtg39PuhVpa/S4UqXD5apqXFbrUH61GpdrX1+p0uFKlfH1ldrXKo0Oqqs1dr282V/hhUClFwJ9vBEdpKwNN77GXpwQH7QO9kGYv8IjB9G6lL4GqK4AqtXGRVsJVJsWtfFHvlptfK2rcmxbpF6AIgBQBBof5f61r68pUwQA3j51MxHak15XF2auDTbqYuvlNRrL9Xs8Cjzwtl2b1Njv9/XYA0RE1AAhBFSaGpRWaq0Gl8vXPZrev9kg4+MtQ6CPFwKU3ghQGh8DLR6NwSZA6YUAhbfla6U3/BVekDHYGBn0xjCirTSGE20FoFXVPTeFF21lXagxBRxz2TWv9VpX75HtJNL6oci0yP2tlysCAN3VxoPN1cu2t0XuD/iFAX7hQEiC3XfVFgxARNQiaXR6lFRoUajSoEilQWG5BsUVWhSWa1Co0qBYZXzU6Jo37k0qAYJ95Qj29UaIrxwhvt4I8jE+hvgZy4NrXwf6eCOwNuz4K73g3dJOOwlhDCqGmtpFV/daV1UbVhoIMOYgU1G/zBRwHEEmB+R+gDzA+Kjwr33tXxsq/AEvpTF8OIped91+V163/yoAAhAGQFNuXOxNIgV8wwD/iLpg4xde+zziutfhgNzX/m1oJgYgIrqlGAwCZepqFJmCjUqDIpUWReUac9gpUmlwuUrX5G36K7zMQebax+DaYBPiK0fQNUEn2FeOAIXXrXN6yaAHKosA1QVAVQCoLtY+XgCuXqoNLbWBRa+zfN3gck194YTL6aVe9Xs6TKeM5H51j3J/K6/96wccL7nj23yzhKg7LddgSFQ1EKBUxgDnFw74hzccbHxCAKlnBnYGICJyOzq9AZWaGlRqr1s0NVDXPq+45vmVKh2KKjQoqu3FqbnR5U215F5SRAYqEBWoRGTtEhWoRERtWVSQEhEBSvjIb+ErnWq0QMXF2nBz7VIbcCouAhWFzgkp15NIAW/f68a2XD/GpYFxL9ef3vFSOGYcjDuTSGr/Pv4Aol3dGrfDAEREDlGjN+BiuQbnLlfhwhUNVFd1qNQaQ0tF7WOl5prnta8rtTXQ2nAlkzUSCRDmrzCHm4jaYGMON0FKRAYoEezr7Taz0jpETTVw5SygOm/Za2MKOBUXjWM5mkIiAwKigcBoIDAGCIw1PvqGATJvYw+LxSK7rlx2zXPv6157AbLr1pfIPLZngTwDAxARNYvBIFBUYZxI79ylKpy7ZJwt+Nxl4/NClQb6JvbENETpLYW/wsu4KL3gJzcOAPZXeMGvtsxfbhwAHBmoMPbgBCkR5q9oOeNohDCenio9AZSdAEpPAmUnjc8vn2laz41McU2ouS7gBMQYH/0jjKGF6BbBAEREVgkhUFpZXRtqjCHn/OXakHPJ2Ktzo5mC5TKp+bLroNqrlPzktcHlmmBj7bmfogUOBm5MtRoo+/2akHOiNvT8bhzb0RBvPyCodW2ouXa5JuD4hra800PU4jEAEbVQGp0exSotiiuMg4RNt0E4d8kYeM5frrrhFVAyqQQxwUrEhRjnl4kL8UXrUONjXKgvwjnPjG0MeqD8vJWQc9J4yqohEikQHA+0ag+EdbjmsQMQEMVwQ2SFWwSg5cuX45VXXkFhYSESExOxbNky9OvXz2pdnU6H7OxsfPDBBygoKEDHjh3x0ksv4d5777Va/x//+Afmzp2LWbNmIScnx4F7QeQerlbrzaHm2sfiax6LVBqoNDU33JZEAkQF1gWc1qG+iAvxQesQX8SFcqZgmwgBaK4AlSXGU1aVRcbxNxWFwKVTtaetfm98nhmf0LqAc23ICW1jHORLRE3m8gC0YcMGZGRkYOXKlUhKSkJOTg5SUlJw/PhxRERE1KufmZmJjz76CO+++y46deqEb775BmPGjMHu3bvRq1cvi7oHDhzA22+/jR49ejhrd25pQ4YMQc+ePc1BMiEhAbNnz8bs2bMbXEcikWDz5s0YPXq0U9p4K1Nra1BcoUWxSoOi2kfza1O4qdCiognBxkThJUVkoBIRAQpEB/sgLsQHcaF1vTnRwUoovDjuo0FCGOeZqSyuDTXFxkV93WtTmb76xtuUyYHQtvVDTlgH46kqIrILlwegpUuXYurUqUhPTwcArFy5Elu2bMHq1asxZ86cevU//PBDPP/88xgxYgQA4Mknn8S3336L1157DR999JG5XmVlJSZMmIB3330X//d//+ecnXFjqamp0Ol02Lp1a733fvjhBwwePBg//fSTTWHxwIED8PPzs2czWwyd3mCeQbisshqXq6pxSV2Ny+pqXKoyPeqMj2rj+1XVTb8M2cdbhohABSIDlAivfYwIVCAiQGEOPBGBSgQqvTzrKiiDoeH5ZITe+FzojfWE4boyvTGw1CszXLNO7evr39dWXBNsrgs3NVdt2wdFkHFAsWnxiwBC4mtDTnsg6DbjFVFE5FAu/ZZVV1cjLy8Pc+fONZdJpVIkJydjz549VtfRarX17hnl4+ODXbt2WZTNmDEDI0eORHJy8g0DkFarhVZb1+18K97wc8qUKXjwwQdx/vx5tG7d2uK9NWvWoE+fPjb3lIWHh9uziR6tusaA85eraoOMzhxkLqmtBBt1dZNOP1njK5dZBBhjoFEgwhxwlIgMVMBf4YRgYzAYJ0vTXAGuXqmdabb28eoVy+dalbH34/rJ8fQNhBlDA5PpiZu7PN5h5P61E8ZFXhNuIuuX+UUA3rznHZE7cGkAKi0thV6vR2RkpEV5ZGQkjh07ZnWdlJQULF26FIMHD0a7du2Qm5uLTZs2Qa+v+7/j9evXIz8/HwcOHGhSO7Kzs7Fw4cLm74gHuO+++xAeHo73338fmZmZ5vLKykps3LgRc+bMwbhx47Bz505cvnwZ7dq1w7x58zBu3LgGt3n9KbATJ05gypQp2L9/P9q2bYvXX3/d0bvldEIIlFRocbSwAscuqnCssAJHL6rwe0kldHrbLvmWSGCeOTjUT44QX7nx0U+OUF/jY6trXof6y+GvsONXVl8D6K65maNGdU1oudxwmDEHndpp9t2BRGa8RNviUWocHHxtmURaW95Y2XXPpTLjlVTX9tr4RxrDjOm1nD2hRJ7G4/pZX3/9dUydOhWdOnWCRCJBu3btkJ6ejtWrVwMAzp07h1mzZmHbtm1Nvrv43LlzkZGRYX6tUqkQFxfX9EYJ4fi7/lrj7dvkqzu8vLwwadIkvP/++3j++efNvQMbN26EXq/HY489ho0bN+K5555DYGAgtmzZgokTJ6Jdu3YNDki/lsFgwAMPPIDIyEjs27cP5eXljY4N8gQanR4niipxtFCFYxcrcKzQGHguqa2P4/CVyxAeoKgLMr5yhPp5WwSaUNPiK0egj3fTb1gphPFOyuorlnebtriJo42vbT110xAvH0AZBPgEA8pg688VgcZBulJZ7SR4VibHk1mZHM/qct0EexIpr3IiIpu5NACFhYVBJpOhqKjIoryoqAhRUVFW1wkPD8fnn38OjUaDsrIyxMTEYM6cOWjbti0AIC8vD8XFxbjjjjvM6+j1euzcuRNvvvkmtFotZDLLQZ0KhQIKxU1cQaGrApbENH/95pp3wab/83z88cfxyiuv4Pvvv8eQIUMAGE9/Pfjgg4iPj8ezzz5rrvvUU0/hm2++wSeffNKkAPTtt9/i2LFj+OabbxATY/xbLFmyBMOHD7dtn1xACIGCK1fNIcfUu3O6VA1r8/hJJUDbcH90igpA5+hAdIoKQKfoQMQEKZt/2qlaDZQXAOXnjJc7lxcYZ+8tL6h7rVPf3I42RCKrnS4/CPAJshJighsPN7z6iIg8kEsDkFwuR+/evZGbm2u+SshgMCA3NxczZ85sdF2lUonY2FjodDp89tlneOSRRwAAQ4cOxS+//GJRNz09HZ06dcJzzz1XL/y0JJ06dcKAAQOwevVqDBkyBCdPnsQPP/yAF198EXq9HkuWLMEnn3yCgoICVFdXQ6vVwte3aXfuPXr0KOLi4szhBwD69+/vqF1ptkptDY4X1vbmXKx7rNBaH5MT6idH5+gAdIoKNAee9hH+UHrb8O+oRlt364Hy88bFHHJqyzRXmr49072RzHeg9m/kdRNu8CiTsweFiFocl58Cy8jIQFpaGvr06YN+/fohJycHarXafFXYpEmTEBsbi+zsbADAvn37UFBQgJ49e6KgoAAvvPACDAYD/v73vwMAAgIC0K1bN4vP8PPzQ6tWreqV2423r7E3xtm8mxZOrjVlyhQ89dRTWL58OdasWYN27drhrrvuwksvvYTXX38dOTk56N69O/z8/DB79mxUVzfhsl03ptMbsP/0JWw7UoQdx4vxR5n1U5XeMgnaRwSgc1QAOpkCT3QAwv0VTevVqboEnP4euHKuLtSYQo66uGmNlQcYZ+wNijXO0hvUuvYxFghsXTfWhLcjICK6aS4PQGPHjkVJSQkWLFiAwsJC9OzZE1u3bjUPjD579iyk19wQT6PRIDMzE6dOnYK/vz9GjBiBDz/8EMHBwS7aAxj/79lDBkE+8sgjmDVrFtatW4d//etfePLJJyGRSPDjjz9i1KhReOyxxwAYe+J+++03dOnSpUnb7dy5M86dO4eLFy8iOtp41+G9e/c6bD8aU6HRYcfxEmw7UoTvjhfXmxcnMlBhDjhdogPRKSoQbcL8IPdqxoR+l/8A9rwFHPyw8XFgMsU1wSbOeshRBtn++URE1CwuD0AAMHPmzAZPee3YscPi9V133YUjR47YtP3rt9GS+fv7Y+zYsZg7dy5UKhUmT54MAOjQoQM+/fRT7N69GyEhIVi6dCmKioqaHICSk5Nx++23Iy0tDa+88gpUKhWef/55B+6JpQtXruLbo0XYdqQIe0+VWVyRFeYvx9BOkUjuEone8SEI9ZPf/AcW5AO73wCOfFF3aXZYRyCqW12wMYeb1oBvK55mIiJyI24RgMi5pkyZglWrVmHEiBHmMTumXrWUlBT4+vpi2rRpGD16NMrLy5u0TalUis2bN2PKlCno168fEhIS8MYbbzR4i5KbJYTA0YsV2HakCNuOFuJwgeXcTW3D/XBPl0gM6xKJnnEhTb/aqjEGA3ByG/DjG8CZa+adanc3MOApoO2fGXKIiDyERAjhJhN5uA+VSoWgoCCUl5cjMDDQ4j2NRoPTp0+jTZs2Tb7Mnhpmy9/z2vE8244UoeBK3WXcEgnQ+7YQ3NPF2NPTLtzffo2s0QI/fwLseRMoqZ2fSuoFdHsIGDATiOpuv88iIqJma+z3+3rsASK3VqHR4fvfasfzHCu2mEFZ6S3FoPbhGNYlEnd3jkCYv50vx756GfjfamDf28ZbHwDGgcp9JgNJ042ntoiIyCMxAJHbuVh+Fd8eKcJ/rYznaeUnx9DOEbinSxQGtQ+Dj9wBV0RdPgPsXQHk/6tu7p2AGODOJ4HeaRysTER0C2AAIrdw9pIa3xw9h/8eKcIvBZbjjtqGGcfz3NMlEr1us9N4HmsuHDIObP71c+ONMAEgoisw8Gmg6wOAlx0GTxMRkVtgACKX0er0KKvUokilwfObDqCgwhg6JBLgjttCkNzZGHraR9hxPM/1hABOfgv8+Drwxw915W2HAAOeNg5w5sBmIqJbDgNQM3HsePNodXqUX9Wh/KoOV3V6CJ0WOr0BUokEd90ejuHdojC0cyTCAxx8e4UaLfDLp8DuZUDJUWOZRAZ0e9B4RVd0D8d+PhERuRQDkI28vb0BAFVVVfDx8XFxazyDtqY29FQZQ4+JBBLIJXoE+Xjji6cGIzTACX/Pq1eAvDXA3pVAZaGxTO4P9J5sHNgcbMNNcImIyGMxANlIJpMhODgYxcXG2xv4+vo2/waYt7DqGj0qtDWouFoDbY1l6PGRSxGg8ILUUI2yinLERYU7NvwIARQdBg6tMw5srq40lgdEG0NP78nGm3oSEVGLwQDUDKY71ZtCEBnVGAy4Wq3HVZ0e1TV1pwglABReUvjIZfDxlqHmqgSXa98LDg42/z3truQ4cHgTcPgzoOxEXXlEF+Nprm4PcWAzEVELxQDUDBKJBNHR0YiIiIBOp3N1c1yqsPwqvv+tBN8fL8HxogpzuUQC9IwLxpDbIzCwfRhCrNx+wtvbGzKZnS9jv3TKGHp+3Wzs9TGRKYAO9xh7e9onc2AzEVELxwB0E2Qymf1/wD1AwZWr+Prni/jql4v46dwVc7lUAtzZthVG9ohGStco+09M2JAr54yB59dNwIWDdeVSL6DdUKDbA0DHEYCy8VlBiYio5WAAoibbergQb+/8HQfPXjGXSSVAUptWGNEjGvd2jXL81VsmFYXGG5Ee/gw4t6+uXCIF2gw2Xs3V6T7AN9Q57SEiIo/CAERN8lneeTz76U8Qwnj2qF9CKO7rEY2UblGICHDSPdHUZcDRL4ynuP7YBcA0zkgCxA8w9vR0HgX4hzunPURE5LEYgOiGvjhUgL/Vhp9x/eLwTPLtiAh0Uui5egU49pUx9JzaUTdDMwC07mvs6ekyCgiMcU57iIjolsAARI36+peLyPjkJxgEMK7fbVg8uhukjroVhYm2Ajj+H2Po+T0X0FfXvRedaLwtRdcxQEi8Y9tBRES3LAYgatB/fy3E0x8fhN4g8FDv1o4LP9pK42XqxceA418DJ/4L1Gjq3o/oYgw93R4AWrWz/+cTEVGLwwBEVn13rBgz1uWjxiAwumcMXnqwx82FHyGAyiKg9Dfj/DylJ4zPS08AqvP167dqXxd6Ijo3/3OJiIisYACienb+VoK/fJQHnV5gZI9ovPpwYtPvwK7XAZdO14ab2oBTWht4tKqG1/MLB8Jurx3X8wAQ1YNz9RARkcMwAJGF3b+XYuq//ofqGgNSukYiZ2xPeMmk9StqyoHSk7Xh5re6Hp1LpwBDjfWNS6RASBtj0AnrUPtY+5yXqxMRkRMxAJHZ/tOXMOX9/0FbY8DQThFYNu4OeJvCj8EA5K0Gfv3cGHZMNxK1xtuvLuCE314XdELbAl5OmieIiIioEQxABADIO3MZ6Wv246pOj7tuD8dbj90BuVdt+FFdAD5/0ngZ+rUCoq/ryaldAmN4+oqIiNwaAxDh5/NXMHn1fqir9RjYvhXentgbCq/aW3z8+jnw71mA5grg5QP8eS4QPwgIaw8og1zZbCIiomZjAGrhDheU47H39qFCW4N+bULx3qS+UHrLAI0K2DoHOLTWWDGmF/DAu8YeHyIiIg/HANSCHStUYeKqfVBpatA7PgSrJ/eFj1wGnN0LbJoGXDljHLg8KAMYMgeQebu6yURERHbBANRCnSiqwIR39+FylQ6JccFYk94X/l4C2P5/wA+vAcIABN8GjHkHiO/v6uYSERHZFQNQC3SqpBLj39uHMnU1usUG4l/p/RCoPgtsmgoU5BkrJY4Dhr/EcT5ERHRLYgBqYc6UqTH+3X0oqdCiU1QAPkzvh6Cja4GtcwFdFaAMBu77p3EyQiIiolsUA1ALcu5SFca/uw+FKg06RPhj3fh2CPn3ZOP9twCgzWBg9EogKNal7SQiInI0BqAW4sKVqxj/3l4UXLmKtuF+2JhcieAPhgDqYkAmB4YuAO6cAUitzPpMRER0i3GLX7vly5cjISEBSqUSSUlJ2L9/f4N1dTodXnzxRbRr1w5KpRKJiYnYunWrRZ3s7Gz07dsXAQEBiIiIwOjRo3H8+HFH74bbKlJpMP7dvTh36So6hsrwVdsvELxpvDH8hHcGpm4HBjzF8ENERC2Gy3/xNmzYgIyMDGRlZSE/Px+JiYlISUlBcXGx1fqZmZl4++23sWzZMhw5cgTTp0/HmDFjcPDgQXOd77//HjNmzMDevXuxbds26HQ6DBs2DGq12lm75TZKKrQY9+5e/FFWhT8HXcRXykz4/rTa+GbSk8C0HUBUd5e2kYiIyNkkQgjhygYkJSWhb9++ePPNNwEABoMBcXFxeOqppzBnzpx69WNiYvD8889jxowZ5rIHH3wQPj4++Oijj6x+RklJCSIiIvD9999j8ODBN2yTSqVCUFAQysvLERgY2Mw9c72ySmP4OVmkwrN+W/Gk2ACJQQf4RwGj3wLaD3V1E4mIiOzGlt9vl44Bqq6uRl5eHubOnWsuk0qlSE5Oxp49e6yuo9VqoVQqLcp8fHywa9euBj+nvLwcABAaav2O41qtFlqt1vxapVI1eR/c1WV1NSa8tw+VRX9go8/b6K3/1fhGp/uA1DcAv1aubSAREZELufQUWGlpKfR6PSIjIy3KIyMjUVho/W7jKSkpWLp0KU6cOAGDwYBt27Zh06ZNuHjxotX6BoMBs2fPxsCBA9GtWzerdbKzsxEUFGRe4uLibm7HXKz8qg4TV+/D7cVb8Y1yDnqLXwG5PzBqOTD2I4YfIiJq8Vw+BshWr7/+Ojp06IBOnTpBLpdj5syZSE9Ph7SBAbwzZszA4cOHsX79+ga3OXfuXJSXl5uXc+fOOar5Dleh0eHJ97bjieIleEO+HAGoAlr3Bab/APR6jHdpJyIigotPgYWFhUEmk6GoqMiivKioCFFRUVbXCQ8Px+effw6NRoOysjLExMRgzpw5aNu2bb26M2fOxFdffYWdO3eidevWDbZDoVBAoVDc3M64AbW2Bi+/vQqvXHoZsbIyCIkMkrv+DvzpWUDGGQ+IiIhMXNoDJJfL0bt3b+Tm5prLDAYDcnNz0b9/4/efUiqViI2NRU1NDT777DOMGjXK/J4QAjNnzsTmzZuxfft2tGnTxmH74E52bHwTCy/NQaykDNrAeEge/6b2JqYMP0RERNdy+S9jRkYG0tLS0KdPH/Tr1w85OTlQq9VIT08HAEyaNAmxsbHIzs4GAOzbtw8FBQXo2bMnCgoK8MILL8BgMODvf/+7eZszZszAunXr8MUXXyAgIMA8nigoKAg+Pj7O30lnqKnGgNPLIJUInIy6D+3T3wYU/q5uFRERkVtyeQAaO3YsSkpKsGDBAhQWFqJnz57YunWreWD02bNnLcb3aDQaZGZm4tSpU/D398eIESPw4YcfIjg42FxnxYoVAIAhQ4ZYfNaaNWswefJkR++Sa/zyCUL0pSgUIfj9ziVoz/BDRETUIJfPA+SOPG4eIIMBWN4PKDuBxbrxGDZ1MfomWL/kn4iI6FZly++3x10FRlYc/xooOwGV8MXH+rvRyk/u6hYRERG5NQYgTycEsGspAOBf+ntQCV+08vf8K9qIiIgciQHI0/2xCyjIg5Ap8X7NvZDLpAhUunxoFxERkVtjAPJ0P+YAAMo6PIRSBKGVvxwSTnZIRETUKAYgT3bxZ+Dkt4BEit/aTwYAhPH0FxER0Q0xAHmyH183PnYdg/PCOHN2K38OgCYiIroRBiBPdek08Osm4/OBs1GqNt7Nnj1AREREN8YA5Kn2vAkIA9A+GYjugdKKagDsASIiImoKBiBPVFkMHPzI+HzgbABAmakHyI89QERERDfCAOSJ9q0EajRAbB8gYRAAoKzS2AMUFsAeICIiohthAPI0GhVw4D3j80GzgdpL3ksrjT1ArdgDREREdEMMQJ4m731AUw606gB0HGkuLjX1AHEQNBER0Q0xAHmSGi2w9y3j80GzAanx8OkNApfMV4HxFBgREdGNMAB5kp83ABUXgYAYoPsj5uIrVdUwCOPzEN4IlYiI6IYYgDyFQV838WH/GYBXXdApUxtPf4X4esNbxkNKRER0I/y19BTHtgBlJwFlENA7zeKt0oraAdAc/0NERNQkDECeQAhg1z+Nz/tNAxQBFm+Xqk0DoHn6i4iIqCkYgDzBHz8AF/IBLx8gaXq9t9kDREREZBsGIE9g6v3p9RjgF1bv7bpZoNkDRERE1BQMQO7uwiHg9+2ARAYMmGm1iuk+YJwDiIiIqGkYgNyd6cqvbg8AIQlWq5h6gHgKjIiIqGkYgNzZpVPAkc+Nz2tvempN3SzQPAVGRETUFAxA7mz3MkAYgPb3AFHdGqxmvg8Ye4CIiIiahAHIXVUUAQfXGp8PeqbRqmXsASIiIrIJA5C72rcC0GuB1v2A+AENVlNra3BVpwfAQdBERERNxQDkjjTlwIFVxueDZgMSSYNVTb0/Sm8pfOUyJzSOiIjI8zEAuaP/rQG0KiCsI3D78EarlprvAq+ApJGgRERERHUYgNyNTgPsfcv4fOAsQNr4IeIs0ERERLZjAHI3P68HKouAwFig+8M3rG66EzxngSYiImo6BiB3YtDXTXzYfybgdeNQY+oB4gBoIiKipnOLALR8+XIkJCRAqVQiKSkJ+/fvb7CuTqfDiy++iHbt2kGpVCIxMRFbt269qW26jaP/Nk5+qAwG7pjUpFVMPUCteAk8ERFRk7k8AG3YsAEZGRnIyspCfn4+EhMTkZKSguLiYqv1MzMz8fbbb2PZsmU4cuQIpk+fjjFjxuDgwYPN3qZbEKLupqdJfwEU/k1azTQJInuAiIiIms7lAWjp0qWYOnUq0tPT0aVLF6xcuRK+vr5YvXq11foffvgh5s2bhxEjRqBt27Z48sknMWLECLz22mvN3qZbOP09cPEQ4OUD9PtLk1ermwWaPUBERERN5dIAVF1djby8PCQnJ5vLpFIpkpOTsWfPHqvraLVaKJVKizIfHx/s2rWr2dt0C6benzsmAX6tmrxa3SzQ7AEiIiJqKpcGoNLSUuj1ekRGRlqUR0ZGorCw0Oo6KSkpWLp0KU6cOAGDwYBt27Zh06ZNuHjxYrO3qdVqoVKpLBanunAQOLUDkMiAATNtWpWnwIiIiGzn8lNgtnr99dfRoUMHdOrUCXK5HDNnzkR6ejqkN5gvpzHZ2dkICgoyL3FxcXZscRPsyjE+dn8ICL6tyavV6A24XKUDwFNgREREtnBpAAoLC4NMJkNRUZFFeVFREaKioqyuEx4ejs8//xxqtRpnzpzBsWPH4O/vj7Zt2zZ7m3PnzkV5ebl5OXfunB32ronKfgeOfGF8PnCWTateqjKe/pJKgBBfBiAiIqKmcmkAksvl6N27N3Jzc81lBoMBubm56N+/f6PrKpVKxMbGoqamBp999hlGjRrV7G0qFAoEBgZaLE6z+w0AAuiQAkR2tWnV0gpjAAr1k0Mm5W0wiIiImsrL1Q3IyMhAWloa+vTpg379+iEnJwdqtRrp6ekAgEmTJiE2NhbZ2dkAgH379qGgoAA9e/ZEQUEBXnjhBRgMBvz9739v8jbdRkUhcGid8fmgZ2xevUzN8T9ERETN4fIANHbsWJSUlGDBggUoLCxEz549sXXrVvMg5rNnz1qM79FoNMjMzMSpU6fg7++PESNG4MMPP0RwcHCTt+k29r4F6KuBuDuB+MZ7vKzhJfBERETNIxFCCFc3wt2oVCoEBQWhvLzccafDNOXAP7sZ7/o+bj3QsfG7vlvz3g+n8H9bjuL+xBi8Ma6XAxpJRETkOWz5/fa4q8BuGQdWGcNPeGfj+J9mKOUcQERERM3CAOQKOg2wd4Xx+cBZQDMv4ecpMCIiouZhAHKFn9YB6mIgsLVx7p9mKqsNQOHsASIiIrIJA5CzGfTAj28Ynw+YCci8m70p0ykw9gARERHZhgHI2Y58AVw+DfiEGO/7dRPKzKfA2ANERERkCwYgZxKi7qanSdMBud9NbEqgVG0aBM0eICIiIlswADnTqe+Awp8Bb1+g37Sb2lSFtgbVNQYAvAqMiIjIVi6fCLFFqboE+LYCuj8C+Ibe1KbKasf/+Cu8oPSW2aN1RERELQYDkDN1fwjoOALQa296U7wEnoiIqPkYgJxN7gvA96Y3Yx4A7ccAREREZCuOAfJQnAWaiIio+RiAPFQpL4EnIiJqNgYgD2UaBB3OMUBEREQ2YwDyUOwBIiIiaj4GIA9VxttgEBERNRsDkIcy9QBxEDQREZHtGIA8VF0AYg8QERGRrRiAPFB1jQEqTQ0A9gARERE1BwOQBypTG3t/vKQSBCq9XdwaIiIiz8MA5IFMA6BD/eSQSiUubg0REZHnYQDyQCUcAE1ERHRTGIA8EC+BJyIiujkMQB7IdCPUcPYAERERNQsDkAeqmwWaPUBERETNwQDkgepOgbEHiIiIqDkYgDwQB0ETERHdHAYgD8RB0ERERDeHAcgDmSZC5CBoIiKi5mEA8jAGg2APEBER0U1iAPIwKo0ONQYBAGjlxx4gIiKi5rA5ACUkJODFF1/E2bNn7dKA5cuXIyEhAUqlEklJSdi/f3+j9XNyctCxY0f4+PggLi4OzzzzDDQajfl9vV6P+fPno02bNvDx8UG7du2waNEiCCHs0l5XM10CH6j0gtyL+ZWIiKg5bP4FnT17NjZt2oS2bdvinnvuwfr166HVapv14Rs2bEBGRgaysrKQn5+PxMREpKSkoLi42Gr9devWYc6cOcjKysLRo0exatUqbNiwAfPmzTPXeemll7BixQq8+eabOHr0KF566SW8/PLLWLZsWbPa6G5Ka09/8QowIiKi5mtWADp06BD279+Pzp0746mnnkJ0dDRmzpyJ/Px8m7a1dOlSTJ06Fenp6ejSpQtWrlwJX19frF692mr93bt3Y+DAgRg/fjwSEhIwbNgwjBs3zqLXaPfu3Rg1ahRGjhyJhIQEPPTQQxg2bNgNe5Y8RRkDEBER0U1r9jmUO+64A2+88QYuXLiArKwsvPfee+jbty969uyJ1atX3/CUU3V1NfLy8pCcnFzXGKkUycnJ2LNnj9V1BgwYgLy8PHOYOXXqFL7++muMGDHCok5ubi5+++03AMBPP/2EXbt2Yfjw4c3dVbfCWaCJiIhunldzV9TpdNi8eTPWrFmDbdu24c4778SUKVNw/vx5zJs3D99++y3WrVvX4PqlpaXQ6/WIjIy0KI+MjMSxY8esrjN+/HiUlpZi0KBBEEKgpqYG06dPtzgFNmfOHKhUKnTq1AkymQx6vR6LFy/GhAkTGmyLVqu1OI2nUqma+mdwujJOgkhERHTTbA5A+fn5WLNmDT7++GNIpVJMmjQJ//znP9GpUydznTFjxqBv3752bSgA7NixA0uWLMFbb72FpKQknDx5ErNmzcKiRYswf/58AMAnn3yCtWvXYt26dejatSsOHTqE2bNnIyYmBmlpaVa3m52djYULF9q9vY5QwkvgiYiIbprNAahv37645557sGLFCowePRre3t716rRp0waPPvpoo9sJCwuDTCZDUVGRRXlRURGioqKsrjN//nxMnDgRTzzxBACge/fuUKvVmDZtGp5//nlIpVL87W9/w5w5c8yf3717d5w5cwbZ2dkNBqC5c+ciIyPD/FqlUiEuLq7R9rtKmfkUGHuAiIiImsvmAHTq1CnEx8c3WsfPzw9r1qxptI5cLkfv3r2Rm5uL0aNHAwAMBgNyc3Mxc+ZMq+tUVVVBKrUctiSTyQDAPOaooToGg6HBtigUCigUnhEoytTGHqBw9gARERE1m80BqLi4GIWFhUhKSrIo37dvH2QyGfr06dPkbWVkZCAtLQ19+vRBv379kJOTA7VajfT0dADApEmTEBsbi+zsbABAamoqli5dil69eplPgc2fPx+pqanmIJSamorFixfjtttuQ9euXXHw4EEsXboUjz/+uK276pZK2QNERER002wOQDNmzMDf//73egGooKAAL730Evbt29fkbY0dOxYlJSVYsGABCgsL0bNnT2zdutU8MPrs2bMWvTmZmZmQSCTIzMxEQUEBwsPDzYHHZNmyZZg/fz7+3//7fyguLkZMTAz+8pe/YMGCBbbuqlviZfBEREQ3TyJsnCLZ398fP//8M9q2bWtRfvr0afTo0QMVFRV2baArqFQqBAUFoby8HIGBga5ujplGp0en+VsBAD+/MAyByvrjr4iIiFoqW36/bZ4HSKFQ1Bu4DAAXL16El1ezr6qnJjCd/pLLpAhQ8G9NRETUXDYHoGHDhmHu3LkoLy83l125cgXz5s3DPffcY9fGkaW6019ySCQSF7eGiIjIc9ncjfDqq69i8ODBiI+PR69evQAAhw4dQmRkJD788EO7N5DqcAA0ERGRfdgcgGJjY/Hzzz9j7dq1+Omnn+Dj44P09HSMGzfO6pxAZD/X9gARERFR8zVrIImfnx+mTZtm77bQDZSwB4iIiMgumj2S9siRIzh79iyqq6styu+///6bbhRZV8bbYBAREdlFs2aCHjNmDH755RdIJBLzDMymQbl6vd6+LSQz0xigcPYAERER3RSbrwKbNWsW2rRpg+LiYvj6+uLXX3/Fzp070adPH+zYscMBTSSTMrXpFBh7gIiIiG6GzT1Ae/bswfbt2xEWFgapVAqpVIpBgwYhOzsbTz/9NA4ePOiIdhI4CzQREZG92NwDpNfrERAQAMB4R/cLFy4AAOLj43H8+HH7to4smC+D92MAIiIiuhk29wB169YNP/30E9q0aYOkpCS8/PLLkMvleOedd+rdHoPsR28QuKTmZfBERET2YHMAyszMhFqtBgC8+OKLuO+++/CnP/0JrVq1woYNG+zeQDK6XFUNQ+1d20L9GICIiIhuhs0BKCUlxfy8ffv2OHbsGC5duoSQkBDensGBTON/Qny94SWz+cwlERERXcOmX1KdTgcvLy8cPnzYojw0NJThx8HKasf/cAA0ERHRzbMpAHl7e+O2227jXD8uUDcLNE9/ERER3Sybz6U8//zzmDdvHi5duuSI9lAD6maBZg8QERHRzbJ5DNCbb76JkydPIiYmBvHx8fDz87N4Pz8/326NozqcBZqIiMh+bA5Ao0ePdkAz6EbMPUC8AoyIiOim2RyAsrKyHNEOugHTbTDCAtgDREREdLN4PbWHKGEPEBERkd3Y3AMklUobveSdV4g5hvkyePYAERER3TSbA9DmzZstXut0Ohw8eBAffPABFi5caLeGUR0hhHkQdBjvA0ZERHTTbA5Ao0aNqlf20EMPoWvXrtiwYQOmTJlil4ZRnapqPTQ6AwDOA0RERGQPdhsDdOeddyI3N9dem6NrmK4A8/GWwU9hc2YlIiKi69glAF29ehVvvPEGYmNj7bE5ug5ngSYiIrIvm7sTrr/pqRACFRUV8PX1xUcffWTXxpER7wNGRERkXzYHoH/+858WAUgqlSI8PBxJSUkICQmxa+PIqLT2FFgYe4CIiIjswuYANHnyZAc0gxpj6gFqxSvAiIiI7MLmMUBr1qzBxo0b65Vv3LgRH3zwgV0aRZbK1LU9QAHsASIiIrIHmwNQdnY2wsLC6pVHRERgyZIldmkUWSphDxAREZFd2RyAzp49izZt2tQrj4+Px9mzZ+3SKLLEWaCJiIjsy+YAFBERgZ9//rle+U8//YRWrVrZ3IDly5cjISEBSqUSSUlJ2L9/f6P1c3Jy0LFjR/j4+CAuLg7PPPMMNBqNRZ2CggI89thjaNWqFXx8fNC9e3f873//s7lt7sI8CJr3ASMiIrILmwdBjxs3Dk8//TQCAgIwePBgAMD333+PWbNm4dFHH7VpWxs2bEBGRgZWrlyJpKQk5OTkICUlBcePH0dERES9+uvWrcOcOXOwevVqDBgwAL/99hsmT54MiUSCpUuXAgAuX76MgQMH4s9//jP+85//IDw8HCdOnPDoK9TMg6B5GTwREZFd2ByAFi1ahD/++ANDhw6Fl5dxdYPBgEmTJtk8Bmjp0qWYOnUq0tPTAQArV67Eli1bsHr1asyZM6de/d27d2PgwIEYP348ACAhIQHjxo3Dvn37zHVeeuklxMXFYc2aNeYya6fsPEWN3oDLVToAvAyeiIjIXmw+BSaXy7FhwwYcP34ca9euxaZNm/D7779j9erVkMub/gNdXV2NvLw8JCcn1zVGKkVycjL27NljdZ0BAwYgLy/PfJrs1KlT+PrrrzFixAhznS+//BJ9+vTBww8/jIiICPTq1Qvvvvtuo23RarVQqVQWi7u4VHsFmFQCBPsyABEREdlDs28s1aFDB3To0KHZH1xaWgq9Xo/IyEiL8sjISBw7dszqOuPHj0dpaSkGDRoEIQRqamowffp0zJs3z1zn1KlTWLFiBTIyMjBv3jwcOHAATz/9NORyOdLS0qxuNzs7223vZG8a/xPqp4BMKrlBbSIiImoKm3uAHnzwQbz00kv1yl9++WU8/PDDdmlUQ3bs2IElS5bgrbfeQn5+PjZt2oQtW7Zg0aJF5joGgwF33HEHlixZgl69emHatGmYOnUqVq5c2eB2586di/LycvNy7tw5h+6HLUrNt8Fg7w8REZG92ByAdu7caXHKyWT48OHYuXNnk7cTFhYGmUyGoqIii/KioiJERUVZXWf+/PmYOHEinnjiCXTv3h1jxozBkiVLkJ2dDYPBAACIjo5Gly5dLNbr3Llzo5foKxQKBAYGWizuokzNG6ESERHZm80BqLKy0upYH29vb5vGzsjlcvTu3Ru5ubnmMoPBgNzcXPTv39/qOlVVVZBKLZssk8kAGG/KCgADBw7E8ePHLer89ttviI+Pb3Lb3EmZ+T5gvAKMiIjIXmwOQN27d8eGDRvqla9fv75ez8uNZGRk4N1338UHH3yAo0eP4sknn4RarTZfFTZp0iTMnTvXXD81NRUrVqzA+vXrcfr0aWzbtg3z589HamqqOQg988wz2Lt3L5YsWYKTJ09i3bp1eOeddzBjxgxbd9UtcBZoIiIi+7N5EPT8+fPxwAMP4Pfff8fdd98NAMjNzcW6devw6aef2rStsWPHoqSkBAsWLEBhYSF69uyJrVu3mgdGnz171qLHJzMzExKJBJmZmSgoKEB4eDhSU1OxePFic52+ffti8+bNmDt3Ll588UW0adMGOTk5mDBhgq276hbMPUC8DxgREZHdSITp3JENtmzZgiVLluDQoUPw8fFBYmIisrKyEBoaim7dujminU6lUqkQFBSE8vJyl48HmrxmP3YcL8HLD/bAI33jXNoWIiIid2bL73ezLoMfOXIkRo4caf6wjz/+GM8++yzy8vKg1+ubs0lqgKkHiIOgiYiI7MfmMUAmO3fuRFpaGmJiYvDaa6/h7rvvxt69e+3ZNsK1l8FzDBAREZG92NQDVFhYiPfffx+rVq2CSqXCI488Aq1Wi88//9zmAdB0Y0II9gARERE5QJN7gFJTU9GxY0f8/PPPyMnJwYULF7Bs2TJHtq3Fq9DWoFpvnN+IPUBERET20+QeoP/85z94+umn8eSTT97ULTCo6UorjKe//BVeUHrLXNwaIiKiW0eTe4B27dqFiooK9O7dG0lJSXjzzTdRWlrqyLa1eGVqnv4iIiJyhCYHoDvvvBPvvvsuLl68iL/85S9Yv349YmJiYDAYsG3bNlRUVDiynS2SqQeIp7+IiIjsy+arwPz8/PD4449j165d+OWXX/DXv/4V//jHPxAREYH777/fEW1ssUpNPUB+7AEiIiKyp2ZfBg8AHTt2xMsvv4zz58/j448/tlebqFaZ6RL4APYAERER2dNNBSATmUyG0aNH48svv7TH5qiWeQ4g9gARERHZlV0CEDlG3X3A2ANERERkTwxAbqyUd4InIiJyCAYgN8ZZoImIiByDAciN8T5gREREjsEA5Ka0NXqoNDUAgDD2ABEREdkVA5CbulQ7B5CXVIIgH28Xt4aIiOjWwgDkpkor6sb/SCQSF7eGiIjo1sIA5KZK1bwCjIiIyFEYgNwU5wAiIiJyHAYgN8VZoImIiByHAchN8T5gREREjsMA5KZKK3kneCIiIkdhAHJT5ttgcBJEIiIiu2MAclPmQdCcBJGIiMjuGIDcFG+DQURE5DgMQG7IYBDmmaAZgIiIiOyPAcgNlV/VocYgAAChHARNRERkdwxAbqisdhboQKUX5F48RERERPbGX1c3VMpZoImIiByKAcgN1c0CzQBERETkCG4RgJYvX46EhAQolUokJSVh//79jdbPyclBx44d4ePjg7i4ODzzzDPQaDRW6/7jH/+ARCLB7NmzHdByx6i7DxjH/xARETmCywPQhg0bkJGRgaysLOTn5yMxMREpKSkoLi62Wn/dunWYM2cOsrKycPToUaxatQobNmzAvHnz6tU9cOAA3n77bfTo0cPRu2FX5kkQ2QNERETkEC4PQEuXLsXUqVORnp6OLl26YOXKlfD19cXq1aut1t+9ezcGDhyI8ePHIyEhAcOGDcO4cePq9RpVVlZiwoQJePfddxESEuKMXbEb820wOAkiERGRQ7g0AFVXVyMvLw/JycnmMqlUiuTkZOzZs8fqOgMGDEBeXp458Jw6dQpff/01RowYYVFvxowZGDlypMW2G6LVaqFSqSwWVyrjJIhEREQO5eXKDy8tLYVer0dkZKRFeWRkJI4dO2Z1nfHjx6O0tBSDBg2CEAI1NTWYPn26xSmw9evXIz8/HwcOHGhSO7Kzs7Fw4cLm74id1c0CzR4gIiIiR3D5KTBb7dixA0uWLMFbb72F/Px8bNq0CVu2bMGiRYsAAOfOncOsWbOwdu1aKJXKJm1z7ty5KC8vNy/nzp1z5C7cUBlngSYiInIol/YAhYWFQSaToaioyKK8qKgIUVFRVteZP38+Jk6ciCeeeAIA0L17d6jVakybNg3PP/888vLyUFxcjDvuuMO8jl6vx86dO/Hmm29Cq9VCJpNZbFOhUEChcJ+wUVrBO8ETERE5kkt7gORyOXr37o3c3FxzmcFgQG5uLvr37291naqqKkills02BRohBIYOHYpffvkFhw4dMi99+vTBhAkTcOjQoXrhx91crdZDXa0HwEHQREREjuLSHiAAyMjIQFpaGvr06YN+/fohJycHarUa6enpAIBJkyYhNjYW2dnZAIDU1FQsXboUvXr1QlJSEk6ePIn58+cjNTUVMpkMAQEB6Natm8Vn+Pn5oVWrVvXK3ZFp/I/cS4oAhcsPDxER0S3J5b+wY8eORUlJCRYsWIDCwkL07NkTW7duNQ+MPnv2rEWPT2ZmJiQSCTIzM1FQUIDw8HCkpqZi8eLFrtoFuzKP//GTQyKRuLg1REREtyaJEEK4uhHuRqVSISgoCOXl5QgMDHTqZ+ceLcKUD/6HHq2D8OXMQU79bCIiIk9my++3x10FdqurmwWa43+IiIgchQHIzZjvBM8rwIiIiByGAcjNmHuAGICIiIgchgHIzZjvBM9L4ImIiByGAcjNlKl5HzAiIiJHYwByM6UVvBM8ERGRozEAuRn2ABERETkeA5Ab0RsELqnZA0RERORoDEBu5HJVNQwCkEiAUF8GICIiIkdhAHIjpivAQnzl8JLx0BARETkKf2XdCGeBJiIicg4GIDdiCkAcAE1ERORYDEBuxHQbDA6AJiIiciwGIDdSxh4gIiIip2AAciO8DQYREZFzMAC5Ed4IlYiIyDkYgNxIqdrUA8QARERE5EgMQG6ktMLUA8RTYERERI7EAOQmhBB19wHzYw8QERGRIzEAuYmqaj00OgMAICyAPUBERESOxADkJkwDoH28ZfCVe7m4NURERLc2BiA3YZoEkb0/REREjscA5Cbq7gPG8T9ERESOxgDkJjgJIhERkfMwALkJ3gaDiIjIeRiA3ETdLNDsASIiInI0BiA3wVmgiYiInIcByE3UzQLNAERERORoDEBuoszUA+THU2BERESOxgDkJsyDoAPYA0RERORobhGAli9fjoSEBCiVSiQlJWH//v2N1s/JyUHHjh3h4+ODuLg4PPPMM9BoNOb3s7Oz0bdvXwQEBCAiIgKjR4/G8ePHHb0bzabTG3C5SgcAaMUeICIiIodzeQDasGEDMjIykJWVhfz8fCQmJiIlJQXFxcVW669btw5z5sxBVlYWjh49ilWrVmHDhg2YN2+euc7333+PGTNmYO/evdi2bRt0Oh2GDRsGtVrtrN2yyeXa019SCRDiywBERETkaBIhhHBlA5KSktC3b1+8+eabAACDwYC4uDg89dRTmDNnTr36M2fOxNGjR5Gbm2su++tf/4p9+/Zh165dVj+jpKQEERER+P777zF48OAbtkmlUiEoKAjl5eUIDAxs5p413a8XyjHyjV0I81fgf5nJDv88IiKiW5Etv98u7QGqrq5GXl4ekpPrfvSlUimSk5OxZ88eq+sMGDAAeXl55tNkp06dwtdff40RI0Y0+Dnl5eUAgNDQUKvva7VaqFQqi8WZOAs0ERGRc7n0tuOlpaXQ6/WIjIy0KI+MjMSxY8esrjN+/HiUlpZi0KBBEEKgpqYG06dPtzgFdi2DwYDZs2dj4MCB6Natm9U62dnZWLhw4c3tzE0o5SzQRERETuXyMUC22rFjB5YsWYK33noL+fn52LRpE7Zs2YJFixZZrT9jxgwcPnwY69evb3Cbc+fORXl5uXk5d+6co5pvlakHiLNAExEROYdLe4DCwsIgk8lQVFRkUV5UVISoqCir68yfPx8TJ07EE088AQDo3r071Go1pk2bhueffx5SaV2mmzlzJr766ivs3LkTrVu3brAdCoUCCoXrel9K1ewBIiIiciaX9gDJ5XL07t3bYkCzwWBAbm4u+vfvb3Wdqqoqi5ADADKZDABgGs8thMDMmTOxefNmbN++HW3atHHQHthHaQV7gIiIiJzJpT1AAJCRkYG0tDT06dMH/fr1Q05ODtRqNdLT0wEAkyZNQmxsLLKzswEAqampWLp0KXr16oWkpCScPHkS8+fPR2pqqjkIzZgxA+vWrcMXX3yBgIAAFBYWAgCCgoLg4+Pjmh1tRBl7gIiIiJzK5QFo7NixKCkpwYIFC1BYWIiePXti69at5oHRZ8+etejxyczMhEQiQWZmJgoKChAeHo7U1FQsXrzYXGfFihUAgCFDhlh81po1azB58mSH75Ot6gZBsweIiIjIGVw+D5A7cvY8QP2zc3GxXIMvZgxEYlywwz+PiIjoVuQx8wCRcbySeR4g3geMiIjIKRiAXEylqUG13gCA9wEjIiJyFgYgFzPdBT5A4QWlt8zFrSEiImoZGIBcrJSTIBIRETkdA5CLmXqAWvESeCIiIqdhAHKxUjVvhEpERORsDEAuVlrBHiAiIiJnYwByMc4CTURE5HwMQC5mug8YT4ERERE5DwOQi5l6gFr5sQeIiIjIWRiAXMw8CzR7gIiIiJyGAcjFSngZPBERkdMxALmQtkaPCk0NACCcAYiIiMhpGIBcyHT6y0sqQaCPl4tbQ0RE1HIwALlQ2TW3wZBIJC5uDRERUcvBAORCpZwDiIiIyCUYgFyIs0ATERG5BgOQC5XxPmBEREQuwQDkQqYeIJ4CIyIici4GIBcy9QC18mMPEBERkTMxALlQaSV7gIiIiFyBAciFSq+5DJ6IiIichwHIhcrYA0REROQSDEAuYjCIa64CYwAiIiJyJgYgFym/qoPeIAAAoRwETURE5FQMQC5SVjsLdJCPN+RePAxERETOxF9eFymp4ABoIiIiV2EAcpEy3geMiIjIZRiAXKRuFmj2ABERETkbA5CL8AowIiIi13GLALR8+XIkJCRAqVQiKSkJ+/fvb7R+Tk4OOnbsCB8fH8TFxeGZZ56BRqO5qW06m2kW6FZ+DEBERETO5vIAtGHDBmRkZCArKwv5+flITExESkoKiouLrdZft24d5syZg6ysLBw9ehSrVq3Chg0bMG/evGZv0xU4CzQREZHruDwALV26FFOnTkV6ejq6dOmClStXwtfXF6tXr7Zaf/fu3Rg4cCDGjx+PhIQEDBs2DOPGjbPo4bF1m67AWaCJiIhcx6UBqLq6Gnl5eUhOTjaXSaVSJCcnY8+ePVbXGTBgAPLy8syB59SpU/j6668xYsSIZm9Tq9VCpVJZLI5m6gHiIGgiIiLn83Llh5eWlkKv1yMyMtKiPDIyEseOHbO6zvjx41FaWopBgwZBCIGamhpMnz7dfAqsOdvMzs7GwoUL7bBHTcceICIiItdx+SkwW+3YsQNLlizBW2+9hfz8fGzatAlbtmzBokWLmr3NuXPnory83LycO3fOji2u72q1HupqPQCOASIiInIFl/YAhYWFQSaToaioyKK8qKgIUVFRVteZP38+Jk6ciCeeeAIA0L17d6jVakybNg3PP/98s7apUCigUDivJ8Z0BZjcSwp/hUsPARERUYvk0h4guVyO3r17Izc311xmMBiQm5uL/v37W12nqqoKUqlls2UyGQBACNGsbTqbaQ6gcH8FJBKJi1tDRETU8ri8+yEjIwNpaWno06cP+vXrh5ycHKjVaqSnpwMAJk2ahNjYWGRnZwMAUlNTsXTpUvTq1QtJSUk4efIk5s+fj9TUVHMQutE2Xc00CzRPfxEREbmGywPQ2LFjUVJSggULFqCwsBA9e/bE1q1bzYOYz549a9Hjk5mZCYlEgszMTBQUFCA8PBypqalYvHhxk7fparwPGBERkWtJhBDC1Y1wNyqVCkFBQSgvL0dgYKDdt7/8u5N45ZvjeLh3a7zycKLdt09ERNQS2fL77XFXgd0KzLfBYA8QERGRSzAAuUAZJ0EkIiJyKQYgFyjlJIhEREQuxQDkAnU9QAxARERErsAA5AJ1Y4B4CoyIiMgVGICcTG8QuFRl7AFiACIiInINBiAnu1xVDSEAiQQI9WUAIiIicgUGICcznf4K8ZXDS8Y/PxERkSvwF9jJeAk8ERGR6zEAOZl5ALQfrwAjIiJyFQYgJyut5ABoIiIiV2MAcrIyToJIRETkcgxATlY3CzR7gIiIiFyFAcjJOAs0ERGR6zEAORnvBE9EROR6DEBOVsrL4ImIiFyOAciJhBAoU3MQNBERkasxADmRuloPjc4AgJfBExERuRIDkBOZLoH3lcvgK/dycWuIiIhaLgYgJ6obAM3eHyIiIldiAHKiUl4CT0RE5BYYgJxIo9PDTy7jfcCIiIhcjANRnGhUz1iM6hmLGr3B1U0hIiJq0dgD5AJeMv7ZiYiIXIm/xERERNTiMAARERFRi8MARERERC0OAxARERG1OAxARERE1OIwABEREVGL4xYBaPny5UhISIBSqURSUhL279/fYN0hQ4ZAIpHUW0aOHGmuU1lZiZkzZ6J169bw8fFBly5dsHLlSmfsChEREXkAlwegDRs2ICMjA1lZWcjPz0diYiJSUlJQXFxstf6mTZtw8eJF83L48GHIZDI8/PDD5joZGRnYunUrPvroIxw9ehSzZ8/GzJkz8eWXXzprt4iIiMiNuTwALV26FFOnTkV6erq5p8bX1xerV6+2Wj80NBRRUVHmZdu2bfD19bUIQLt370ZaWhqGDBmChIQETJs2DYmJiY32LBEREVHL4dIAVF1djby8PCQnJ5vLpFIpkpOTsWfPniZtY9WqVXj00Ufh5+dnLhswYAC+/PJLFBQUQAiB7777Dr/99huGDRtmdRtarRYqlcpiISIioluXSwNQaWkp9Ho9IiMjLcojIyNRWFh4w/X379+Pw4cP44knnrAoX7ZsGbp06YLWrVtDLpfj3nvvxfLlyzF48GCr28nOzkZQUJB5iYuLa/5OERERkdtz+Smwm7Fq1Sp0794d/fr1syhftmwZ9u7diy+//BJ5eXl47bXXMGPGDHz77bdWtzN37lyUl5ebl3Pnzjmj+UREROQiLr0bfFhYGGQyGYqKiizKi4qKEBUV1ei6arUa69evx4svvmhRfvXqVcybNw+bN282XxnWo0cPHDp0CK+++qrF6TYThUIBhUJxk3tDREREnsKlAUgul6N3797Izc3F6NGjAQAGgwG5ubmYOXNmo+tu3LgRWq0Wjz32mEW5TqeDTqeDVGrZuSWTyWAwGJrULiEEAHAsEBERkQcx/W6bfscbJVxs/fr1QqFQiPfff18cOXJETJs2TQQHB4vCwkIhhBATJ04Uc+bMqbfeoEGDxNixY61u86677hJdu3YV3333nTh16pRYs2aNUCqV4q233mpSm86dOycAcOHChQsXLlw8cDl37twNf+td2gMEAGPHjkVJSQkWLFiAwsJC9OzZE1u3bjUPjD579my93pzjx49j165d+O9//2t1m+vXr8fcuXMxYcIEXLp0CfHx8Vi8eDGmT5/epDbFxMTg3LlzCAgIgEQiubkdvI5KpUJcXBzOnTuHwMBAu27b3XBfb10taX+5r7eulrS/LWVfhRCoqKhATEzMDetKhGhKPxHZi0qlQlBQEMrLy2/pf4QA9/VW1pL2l/t662pJ+9uS9rWpPPoqMCIiIqLmYAAiIiKiFocByMkUCgWysrJaxGX33NdbV0vaX+7rrasl7W9L2tem4hggIiIianHYA0REREQtDgMQERERtTgMQERERNTiMAARERFRi8MA5ADLly9HQkIClEolkpKSsH///kbrb9y4EZ06dYJSqUT37t3x9ddfO6mlzZednY2+ffsiICAAERERGD16NI4fP97oOu+//z4kEonFolQqndTi5nvhhRfqtbtTp06NruOJx9QkISGh3v5KJBLMmDHDan1POq47d+5EamoqYmJiIJFI8Pnnn1u8L4TAggULEB0dDR8fHyQnJ+PEiRM33K6t33lnaWx/dTodnnvuOXTv3h1+fn6IiYnBpEmTcOHChUa32ZzvgzPc6NhOnjy5XrvvvffeG27XHY/tjfbV2vdXIpHglVdeaXCb7npcHYkByM42bNiAjIwMZGVlIT8/H4mJiUhJSUFxcbHV+rt378a4ceMwZcoUHDx4EKNHj8bo0aNx+PBhJ7fcNt9//z1mzJiBvXv3Ytu2bdDpdBg2bBjUanWj6wUGBuLixYvm5cyZM05q8c3p2rWrRbt37drVYF1PPaYmBw4csNjXbdu2AQAefvjhBtfxlOOqVquRmJiI5cuXW33/5ZdfxhtvvIGVK1di37598PPzQ0pKCjQaTYPbtPU770yN7W9VVRXy8/Mxf/585OfnY9OmTTh+/Djuv//+G27Xlu+Ds9zo2ALAvffea9Hujz/+uNFtuuuxvdG+XruPFy9exOrVqyGRSPDggw82ul13PK4O1aS7g1KT9evXT8yYMcP8Wq/Xi5iYGJGdnW21/iOPPCJGjhxpUZaUlCT+8pe/OLSd9lZcXCwAiO+//77BOmvWrBFBQUHOa5SdZGVlicTExCbXv1WOqcmsWbNEu3bthMFgsPq+px5XAGLz5s3m1waDQURFRYlXXnnFXHblyhWhUCjExx9/3OB2bP3Ou8r1+2vN/v37BQBx5syZBuvY+n1wBWv7mpaWJkaNGmXTdjzh2DbluI4aNUrcfffdjdbxhONqb+wBsqPq6mrk5eUhOTnZXCaVSpGcnIw9e/ZYXWfPnj0W9QEgJSWlwfruqry8HAAQGhraaL3KykrEx8cjLi4Oo0aNwq+//uqM5t20EydOICYmBm3btsWECRNw9uzZBuveKscUMP6b/uijj/D44483emNgTz2u1zp9+jQKCwstjl1QUBCSkpIaPHbN+c67s/LyckgkEgQHBzdaz5bvgzvZsWMHIiIi0LFjRzz55JMoKytrsO6tcmyLioqwZcsWTJky5YZ1PfW4NhcDkB2VlpZCr9eb72RvEhkZicLCQqvrFBYW2lTfHRkMBsyePRsDBw5Et27dGqzXsWNHrF69Gl988QU++ugjGAwGDBgwAOfPn3dia22XlJSE999/H1u3bsWKFStw+vRp/OlPf0JFRYXV+rfCMTX5/PPPceXKFUyePLnBOp56XK9nOj62HLvmfOfdlUajwXPPPYdx48Y1erNMW78P7uLee+/Fv/71L+Tm5uKll17C999/j+HDh0Ov11utf6sc2w8++AABAQF44IEHGq3nqcf1Zni5ugHk+WbMmIHDhw/f8Hxx//790b9/f/PrAQMGoHPnznj77bexaNEiRzez2YYPH25+3qNHDyQlJSE+Ph6ffPJJk/6vypOtWrUKw4cPR0xMTIN1PPW4Uh2dTodHHnkEQgisWLGi0bqe+n149NFHzc+7d++OHj16oF27dtixYweGDh3qwpY51urVqzFhwoQbXpjgqcf1ZrAHyI7CwsIgk8lQVFRkUV5UVISoqCir60RFRdlU393MnDkTX331Fb777ju0bt3apnW9vb3Rq1cvnDx50kGtc4zg4GDcfvvtDbbb04+pyZkzZ/Dtt9/iiSeesGk9Tz2upuNjy7Frznfe3ZjCz5kzZ7Bt27ZGe3+sudH3wV21bdsWYWFhDbb7Vji2P/zwA44fP27zdxjw3ONqCwYgO5LL5ejduzdyc3PNZQaDAbm5uRb/h3yt/v37W9QHgG3btjVY310IITBz5kxs3rwZ27dvR5s2bWzehl6vxy+//ILo6GgHtNBxKisr8fvvvzfYbk89ptdbs2YNIiIiMHLkSJvW89Tj2qZNG0RFRVkcO5VKhX379jV47JrznXcnpvBz4sQJfPvtt2jVqpXN27jR98FdnT9/HmVlZQ2229OPLWDswe3duzcSExNtXtdTj6tNXD0K+1azfv16oVAoxPvvvy+OHDkipk2bJoKDg0VhYaEQQoiJEyeKOXPmmOv/+OOPwsvLS7z66qvi6NGjIisrS3h7e4tffvnFVbvQJE8++aQICgoSO3bsEBcvXjQvVVVV5jrX7+vChQvFN998I37//XeRl5cnHn30UaFUKsWvv/7qil1osr/+9a9ix44d4vTp0+LHH38UycnJIiwsTBQXFwshbp1jei29Xi9uu+028dxzz9V7z5OPa0VFhTh48KA4ePCgACCWLl0qDh48aL7q6R//+IcIDg4WX3zxhfj555/FqFGjRJs2bcTVq1fN27j77rvFsmXLzK9v9J13pcb2t7q6Wtx///2idevW4tChQxbfY61Wa97G9ft7o++DqzS2rxUVFeLZZ58Ve/bsEadPnxbffvutuOOOO0SHDh2ERqMxb8NTju2N/h0LIUR5ebnw9fUVK1assLoNTzmujsQA5ADLli0Tt912m5DL5aJfv35i79695vfuuusukZaWZlH/k08+EbfffruQy+Wia9euYsuWLU5use0AWF3WrFljrnP9vs6ePdv8d4mMjBQjRowQ+fn5zm+8jcaOHSuio6OFXC4XsbGxYuzYseLkyZPm92+VY3qtb775RgAQx48fr/eeJx/X7777zuq/W9P+GAwGMX/+fBEZGSkUCoUYOnRovb9BfHy8yMrKsihr7DvvSo3t7+nTpxv8Hn/33XfmbVy/vzf6PrhKY/taVVUlhg0bJsLDw4W3t7eIj48XU6dOrRdkPOXY3ujfsRBCvP3228LHx0dcuXLF6jY85bg6kkQIIRzaxURERETkZjgGiIiIiFocBiAiIiJqcRiAiIiIqMVhACIiIqIWhwGIiIiIWhwGICIiImpxGICIiIioxWEAIiJqAolEgs8//9zVzSAiO2EAIiK3N3nyZEgkknrLvffe6+qmEZGH8nJ1A4iImuLee+/FmjVrLMoUCoWLWkNEno49QETkERQKBaKioiyWkJAQAMbTUytWrMDw4cPh4+ODtm3b4tNPP7VY/5dffsHdd98NHx8ftGrVCtOmTUNlZaVFndWrV6Nr165QKBSIjo7GzJkzLd4vLS3FmDFj4Ovriw4dOuDLL7907E4TkcMwABHRLWH+/Pl48MEH8dNPP2HChAl49NFHcfToUQCAWq1GSkoKQkJCcODAAWzcuBHffvutRcBZsWIFZsyYgWnTpuGXX37Bl19+ifbt21t8xsKFC/HII4/g559/xogRIzBhwgRcunTJqftJRHbi6ruxEhHdSFpampDJZMLPz89iWbx4sRBCCABi+vTpFuskJSWJJ598UgghxDvvvCNCQkJEZWWl+f0tW7YIqVRqviN4TEyMeP755xtsAwCRmZlpfl1ZWSkAiP/85z92208ich6OASIij/DnP/8ZK1assCgLDQ01P+/fv7/Fe/3798ehQ4cAAEePHkViYiL8/PzM7w8cOBAGgwHHjx+HRCLBhQsXMHTo0Ebb0KNHD/NzPz8/BAYGori4uLm7REQuxABERB7Bz8+v3ikpe/Hx8WlSPW9vb4vXEokEBoPBEU0iIgfjGCAiuiXs3bu33uvOnTsDADp37oyffvoJarXa/P6PP/4IqVSKjh07IiAgAAkJCcjNzXVqm4nIddgDREQeQavVorCw0KLMy8sLYWFhAICNGzeiT58+GDRoENauXYv9+/dj1apVAIAJEyYgKysLaWlpeOGFF1BSUoKnnnoKEydORGRkJADghRdewPTp0xEREYHhw4ejoqICP/74I5566inn7igROQUDEBF5hK1btyI6OtqirGPHjjh27BgA4xVa69evx//7f/8P0dHR+Pjjj9GlSxcAgK+vL7755hvMmjULffv2ha+vLx588EEsXbrUvK20tDRoNBr885//xLPPPouwsDA89NBDzttBInIqiRBCuLoRREQ3QyKRYPPmzRg9erSrm0JEHoJjgIiIiKjFYQAiIiKiFodjgIjI4/FMPhHZij1ARERE1OIwABEREVGLwwBERERELQ4DEBEREbU4DEBERETU4jAAERERUYvDAEREREQtDgMQERERtTgMQERERNTi/H8ck8bGq4+hAAAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "\n",
    "# Draw accuracy values for training & validation\n",
    "plt.plot(history[\"global_history\"]['multiclassaccuracy'])\n",
    "plt.plot(history[\"global_history\"]['val_multiclassaccuracy'])\n",
    "plt.title('FLModel accuracy')\n",
    "plt.ylabel('Accuracy')\n",
    "plt.xlabel('Epoch')\n",
    "plt.legend(['Train', 'Valid'], loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4678cd74-f29f-4f68-b459-771433c373e6",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
